drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/module.h>
  24 #include <linux/crc32.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/pci.h>
  27 #include <linux/pci_ids.h>
  28 #include <linux/ip.h>
  29 #include <net/ip.h>
  30 #include <linux/ipv6.h>
  31 #include <linux/net_tstamp.h>
  32 #include <linux/if_vlan.h>
  33 #include <linux/firmware.h>
  34 #include <linux/ethtool.h>
  35 #include <linux/ptp_clock_kernel.h>
  36 #include <linux/types.h>
  37 #include <linux/list.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/interrupt.h>
  40 #include "octeon_config.h"
  41 #include "liquidio_common.h"
  42 #include "octeon_droq.h"
  43 #include "octeon_iq.h"
  44 #include "response_manager.h"
  45 #include "octeon_device.h"
  46 #include "octeon_nic.h"
  47 #include "octeon_main.h"
  48 #include "octeon_network.h"
  49 #include "cn66xx_regs.h"
  50 #include "cn66xx_device.h"
  51 #include "cn68xx_regs.h"
  52 #include "cn68xx_device.h"
  53 #include "liquidio_image.h"
  54
  55 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  56 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  57 MODULE_LICENSE("GPL");
  58 MODULE_VERSION(LIQUIDIO_VERSION);
  59 /*(DEBLOBBED)*/
  60
  61 static int ddr_timeout = 10000;
  62 module_param(ddr_timeout, int, 0644);
  63 MODULE_PARM_DESC(ddr_timeout,
  64                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  65
  66 static u32 console_bitmask;
  67 module_param(console_bitmask, int, 0644);
  68 MODULE_PARM_DESC(console_bitmask,
  69                  "Bitmask indicating which consoles have debug output redirected to syslog.");
  70
  71 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  72
  73 static int debug = -1;
  74 module_param(debug, int, 0644);
  75 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  76
  77 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  78 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  79 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  80
  81 static int conf_type;
  82 module_param(conf_type, int, 0);
  83 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
  84
  85 /* Bit mask values for lio->ifstate */
  86 #define   LIO_IFSTATE_DROQ_OPS             0x01
  87 #define   LIO_IFSTATE_REGISTERED           0x02
  88 #define   LIO_IFSTATE_RUNNING              0x04
  89 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
  90
  91 /* Polling interval for determining when NIC application is alive */
  92 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  93
  94 /* runtime link query interval */
  95 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  96
  97 struct liquidio_if_cfg_context {
  98         int octeon_id;
  99
 100         wait_queue_head_t wc;
 101
 102         int cond;
 103 };
 104
 105 struct liquidio_if_cfg_resp {
 106         u64 rh;
 107         struct liquidio_if_cfg_info cfg_info;
 108         u64 status;
 109 };
 110
 111 struct oct_link_status_resp {
 112         u64 rh;
 113         struct oct_link_info link_info;
 114         u64 status;
 115 };
 116
 117 struct oct_timestamp_resp {
 118         u64 rh;
 119         u64 timestamp;
 120         u64 status;
 121 };
 122
 123 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 124
 125 union tx_info {
 126         u64 u64;
 127         struct {
 128 #ifdef __BIG_ENDIAN_BITFIELD
 129                 u16 gso_size;
 130                 u16 gso_segs;
 131                 u32 reserved;
 132 #else
 133                 u32 reserved;
 134                 u16 gso_segs;
 135                 u16 gso_size;
 136 #endif
 137         } s;
 138 };
 139
 140 /** Octeon device properties to be used by the NIC module.
 141  * Each octeon device in the system will be represented
 142  * by this structure in the NIC module.
 143  */
 144
 145 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 146
 147 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 148 #define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE)
 149
 150 /** Structure of a node in list of gather components maintained by
 151  * NIC driver for each network device.
 152  */
 153 struct octnic_gather {
 154         /** List manipulation. Next and prev pointers. */
 155         struct list_head list;
 156
 157         /** Size of the gather component at sg in bytes. */
 158         int sg_size;
 159
 160         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 161         int adjust;
 162
 163         /** Gather component that can accommodate max sized fragment list
 164          *  received from the IP layer.
 165          */
 166         struct octeon_sg_entry *sg;
 167 };
 168
 169 /** This structure is used by NIC driver to store information required
 170  * to free the sk_buff when the packet has been fetched by Octeon.
 171  * Bytes offset below assume worst-case of a 64-bit system.
 172  */
 173 struct octnet_buf_free_info {
 174         /** Bytes 1-8.  Pointer to network device private structure. */
 175         struct lio *lio;
 176
 177         /** Bytes 9-16.  Pointer to sk_buff. */
 178         struct sk_buff *skb;
 179
 180         /** Bytes 17-24.  Pointer to gather list. */
 181         struct octnic_gather *g;
 182
 183         /** Bytes 25-32. Physical address of skb->data or gather list. */
 184         u64 dptr;
 185
 186         /** Bytes 33-47. Piggybacked soft command, if any */
 187         struct octeon_soft_command *sc;
 188 };
 189
 190 struct handshake {
 191         struct completion init;
 192         struct completion started;
 193         struct pci_dev *pci_dev;
 194         int init_ok;
 195         int started_ok;
 196 };
 197
 198 struct octeon_device_priv {
 199         /** Tasklet structures for this device. */
 200         struct tasklet_struct droq_tasklet;
 201         unsigned long napi_mask;
 202 };
 203
 204 static int octeon_device_init(struct octeon_device *);
 205 static void liquidio_remove(struct pci_dev *pdev);
 206 static int liquidio_probe(struct pci_dev *pdev,
 207                           const struct pci_device_id *ent);
 208
 209 static struct handshake handshake[MAX_OCTEON_DEVICES];
 210 static struct completion first_stage;
 211
 212 static void octeon_droq_bh(unsigned long pdev)
 213 {
 214         int q_no;
 215         int reschedule = 0;
 216         struct octeon_device *oct = (struct octeon_device *)pdev;
 217         struct octeon_device_priv *oct_priv =
 218                 (struct octeon_device_priv *)oct->priv;
 219
 220         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
 221         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES; q_no++) {
 222                 if (!(oct->io_qmask.oq & (1UL << q_no)))
 223                         continue;
 224                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 225                                                           MAX_PACKET_BUDGET);
 226         }
 227
 228         if (reschedule)
 229                 tasklet_schedule(&oct_priv->droq_tasklet);
 230 }
 231
 232 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 233 {
 234         struct octeon_device_priv *oct_priv =
 235                 (struct octeon_device_priv *)oct->priv;
 236         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 237         int i;
 238
 239         do {
 240                 pending_pkts = 0;
 241
 242                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
 243                         if (!(oct->io_qmask.oq & (1UL << i)))
 244                                 continue;
 245                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct,
 246                                                                  oct->droq[i]);
 247                 }
 248                 if (pkt_cnt > 0) {
 249                         pending_pkts += pkt_cnt;
 250                         tasklet_schedule(&oct_priv->droq_tasklet);
 251                 }
 252                 pkt_cnt = 0;
 253                 schedule_timeout_uninterruptible(1);
 254
 255         } while (retry-- && pending_pkts);
 256
 257         return pkt_cnt;
 258 }
 259
 260 void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl,
 261                                         unsigned int bytes_compl)
 262 {
 263         struct netdev_queue *netdev_queue = txq;
 264
 265         netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl);
 266 }
 267
 268 void octeon_update_tx_completion_counters(void *buf, int reqtype,
 269                                           unsigned int *pkts_compl,
 270                                           unsigned int *bytes_compl)
 271 {
 272         struct octnet_buf_free_info *finfo;
 273         struct sk_buff *skb = NULL;
 274         struct octeon_soft_command *sc;
 275
 276         switch (reqtype) {
 277         case REQTYPE_NORESP_NET:
 278         case REQTYPE_NORESP_NET_SG:
 279                 finfo = buf;
 280                 skb = finfo->skb;
 281                 break;
 282
 283         case REQTYPE_RESP_NET_SG:
 284         case REQTYPE_RESP_NET:
 285                 sc = buf;
 286                 skb = sc->callback_arg;
 287                 break;
 288
 289         default:
 290                 return;
 291         }
 292
 293         (*pkts_compl)++;
 294         *bytes_compl += skb->len;
 295 }
 296
 297 void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
 298 {
 299         struct octnet_buf_free_info *finfo;
 300         struct sk_buff *skb;
 301         struct octeon_soft_command *sc;
 302         struct netdev_queue *txq;
 303
 304         switch (reqtype) {
 305         case REQTYPE_NORESP_NET:
 306         case REQTYPE_NORESP_NET_SG:
 307                 finfo = buf;
 308                 skb = finfo->skb;
 309                 break;
 310
 311         case REQTYPE_RESP_NET_SG:
 312         case REQTYPE_RESP_NET:
 313                 sc = buf;
 314                 skb = sc->callback_arg;
 315                 break;
 316
 317         default:
 318                 return;
 319         }
 320
 321         txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
 322         netdev_tx_sent_queue(txq, skb->len);
 323 }
 324
 325 int octeon_console_debug_enabled(u32 console)
 326 {
 327         return (console_bitmask >> (console)) & 0x1;
 328 }
 329
 330 /**
 331  * \brief Forces all IO queues off on a given device
 332  * @param oct Pointer to Octeon device
 333  */
 334 static void force_io_queues_off(struct octeon_device *oct)
 335 {
 336         if ((oct->chip_id == OCTEON_CN66XX) ||
 337             (oct->chip_id == OCTEON_CN68XX)) {
 338                 /* Reset the Enable bits for Input Queues. */
 339                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 340
 341                 /* Reset the Enable bits for Output Queues. */
 342                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 343         }
 344 }
 345
 346 /**
 347  * \brief wait for all pending requests to complete
 348  * @param oct Pointer to Octeon device
 349  *
 350  * Called during shutdown sequence
 351  */
 352 static int wait_for_pending_requests(struct octeon_device *oct)
 353 {
 354         int i, pcount = 0;
 355
 356         for (i = 0; i < 100; i++) {
 357                 pcount =
 358                         atomic_read(&oct->response_list
 359                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 360                 if (pcount)
 361                         schedule_timeout_uninterruptible(HZ / 10);
 362                  else
 363                         break;
 364         }
 365
 366         if (pcount)
 367                 return 1;
 368
 369         return 0;
 370 }
 371
 372 /**
 373  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 374  * @param oct Pointer to Octeon device
 375  */
 376 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 377 {
 378         int i;
 379
 380         /* Disable the input and output queues now. No more packets will
 381          * arrive from Octeon, but we should wait for all packet processing
 382          * to finish.
 383          */
 384         force_io_queues_off(oct);
 385
 386         /* To allow for in-flight requests */
 387         schedule_timeout_uninterruptible(100);
 388
 389         if (wait_for_pending_requests(oct))
 390                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 391
 392         /* Force all requests waiting to be fetched by OCTEON to complete. */
 393         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
 394                 struct octeon_instr_queue *iq;
 395
 396                 if (!(oct->io_qmask.iq & (1UL << i)))
 397                         continue;
 398                 iq = oct->instr_queue[i];
 399
 400                 if (atomic_read(&iq->instr_pending)) {
 401                         spin_lock_bh(&iq->lock);
 402                         iq->fill_cnt = 0;
 403                         iq->octeon_read_index = iq->host_write_index;
 404                         iq->stats.instr_processed +=
 405                                 atomic_read(&iq->instr_pending);
 406                         lio_process_iq_request_list(oct, iq);
 407                         spin_unlock_bh(&iq->lock);
 408                 }
 409         }
 410
 411         /* Force all pending ordered list requests to time out. */
 412         lio_process_ordered_list(oct, 1);
 413
 414         /* We do not need to wait for output queue packets to be processed. */
 415 }
 416
 417 /**
 418  * \brief Cleanup PCI AER uncorrectable error status
 419  * @param dev Pointer to PCI device
 420  */
 421 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 422 {
 423         int pos = 0x100;
 424         u32 status, mask;
 425
 426         pr_info("%s :\n", __func__);
 427
 428         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 429         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 430         if (dev->error_state == pci_channel_io_normal)
 431                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 432         else
 433                 status &= mask;         /* Clear corresponding fatal bits */
 434         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 435 }
 436
 437 /**
 438  * \brief Stop all PCI IO to a given device
 439  * @param dev Pointer to Octeon device
 440  */
 441 static void stop_pci_io(struct octeon_device *oct)
 442 {
 443         /* No more instructions will be forwarded. */
 444         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 445
 446         pci_disable_device(oct->pci_dev);
 447
 448         /* Disable interrupts  */
 449         oct->fn_list.disable_interrupt(oct->chip);
 450
 451         pcierror_quiesce_device(oct);
 452
 453         /* Release the interrupt line */
 454         free_irq(oct->pci_dev->irq, oct);
 455
 456         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 457                 pci_disable_msi(oct->pci_dev);
 458
 459         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 460                 lio_get_state_string(&oct->status));
 461
 462         /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
 463         /* making it a common function for all OCTEON models */
 464         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 465 }
 466
 467 /**
 468  * \brief called when PCI error is detected
 469  * @param pdev Pointer to PCI device
 470  * @param state The current pci connection state
 471  *
 472  * This function is called after a PCI bus error affecting
 473  * this device has been detected.
 474  */
 475 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 476                                                      pci_channel_state_t state)
 477 {
 478         struct octeon_device *oct = pci_get_drvdata(pdev);
 479
 480         /* Non-correctable Non-fatal errors */
 481         if (state == pci_channel_io_normal) {
 482                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 483                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 484                 return PCI_ERS_RESULT_CAN_RECOVER;
 485         }
 486
 487         /* Non-correctable Fatal errors */
 488         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 489         stop_pci_io(oct);
 490
 491         /* Always return a DISCONNECT. There is no support for recovery but only
 492          * for a clean shutdown.
 493          */
 494         return PCI_ERS_RESULT_DISCONNECT;
 495 }
 496
 497 /**
 498  * \brief mmio handler
 499  * @param pdev Pointer to PCI device
 500  */
 501 static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev *pdev)
 502 {
 503         /* We should never hit this since we never ask for a reset for a Fatal
 504          * Error. We always return DISCONNECT in io_error above.
 505          * But play safe and return RECOVERED for now.
 506          */
 507         return PCI_ERS_RESULT_RECOVERED;
 508 }
 509
 510 /**
 511  * \brief called after the pci bus has been reset.
 512  * @param pdev Pointer to PCI device
 513  *
 514  * Restart the card from scratch, as if from a cold-boot. Implementation
 515  * resembles the first-half of the octeon_resume routine.
 516  */
 517 static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev *pdev)
 518 {
 519         /* We should never hit this since we never ask for a reset for a Fatal
 520          * Error. We always return DISCONNECT in io_error above.
 521          * But play safe and return RECOVERED for now.
 522          */
 523         return PCI_ERS_RESULT_RECOVERED;
 524 }
 525
 526 /**
 527  * \brief called when traffic can start flowing again.
 528  * @param pdev Pointer to PCI device
 529  *
 530  * This callback is called when the error recovery driver tells us that
 531  * its OK to resume normal operation. Implementation resembles the
 532  * second-half of the octeon_resume routine.
 533  */
 534 static void liquidio_pcie_resume(struct pci_dev *pdev)
 535 {
 536         /* Nothing to be done here. */
 537 }
 538
 539 #ifdef CONFIG_PM
 540 /**
 541  * \brief called when suspending
 542  * @param pdev Pointer to PCI device
 543  * @param state state to suspend to
 544  */
 545 static int liquidio_suspend(struct pci_dev *pdev, pm_message_t state)
 546 {
 547         return 0;
 548 }
 549
 550 /**
 551  * \brief called when resuming
 552  * @param pdev Pointer to PCI device
 553  */
 554 static int liquidio_resume(struct pci_dev *pdev)
 555 {
 556         return 0;
 557 }
 558 #endif
 559
 560 /* For PCI-E Advanced Error Recovery (AER) Interface */
 561 static const struct pci_error_handlers liquidio_err_handler = {
 562         .error_detected = liquidio_pcie_error_detected,
 563         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 564         .slot_reset     = liquidio_pcie_slot_reset,
 565         .resume         = liquidio_pcie_resume,
 566 };
 567
 568 static const struct pci_device_id liquidio_pci_tbl[] = {
 569         {       /* 68xx */
 570                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 571         },
 572         {       /* 66xx */
 573                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 574         },
 575         {
 576                 0, 0, 0, 0, 0, 0, 0
 577         }
 578 };
 579 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 580
 581 static struct pci_driver liquidio_pci_driver = {
 582         .name           = "LiquidIO",
 583         .id_table       = liquidio_pci_tbl,
 584         .probe          = liquidio_probe,
 585         .remove         = liquidio_remove,
 586         .err_handler    = &liquidio_err_handler,    /* For AER */
 587
 588 #ifdef CONFIG_PM
 589         .suspend        = liquidio_suspend,
 590         .resume         = liquidio_resume,
 591 #endif
 592
 593 };
 594
 595 /**
 596  * \brief register PCI driver
 597  */
 598 static int liquidio_init_pci(void)
 599 {
 600         return pci_register_driver(&liquidio_pci_driver);
 601 }
 602
 603 /**
 604  * \brief unregister PCI driver
 605  */
 606 static void liquidio_deinit_pci(void)
 607 {
 608         pci_unregister_driver(&liquidio_pci_driver);
 609 }
 610
 611 /**
 612  * \brief check interface state
 613  * @param lio per-network private data
 614  * @param state_flag flag state to check
 615  */
 616 static inline int ifstate_check(struct lio *lio, int state_flag)
 617 {
 618         return atomic_read(&lio->ifstate) & state_flag;
 619 }
 620
 621 /**
 622  * \brief set interface state
 623  * @param lio per-network private data
 624  * @param state_flag flag state to set
 625  */
 626 static inline void ifstate_set(struct lio *lio, int state_flag)
 627 {
 628         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
 629 }
 630
 631 /**
 632  * \brief clear interface state
 633  * @param lio per-network private data
 634  * @param state_flag flag state to clear
 635  */
 636 static inline void ifstate_reset(struct lio *lio, int state_flag)
 637 {
 638         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 639 }
 640
 641 /**
 642  * \brief Stop Tx queues
 643  * @param netdev network device
 644  */
 645 static inline void txqs_stop(struct net_device *netdev)
 646 {
 647         if (netif_is_multiqueue(netdev)) {
 648                 int i;
 649
 650                 for (i = 0; i < netdev->num_tx_queues; i++)
 651                         netif_stop_subqueue(netdev, i);
 652         } else {
 653                 netif_stop_queue(netdev);
 654         }
 655 }
 656
 657 /**
 658  * \brief Start Tx queues
 659  * @param netdev network device
 660  */
 661 static inline void txqs_start(struct net_device *netdev)
 662 {
 663         if (netif_is_multiqueue(netdev)) {
 664                 int i;
 665
 666                 for (i = 0; i < netdev->num_tx_queues; i++)
 667                         netif_start_subqueue(netdev, i);
 668         } else {
 669                 netif_start_queue(netdev);
 670         }
 671 }
 672
 673 /**
 674  * \brief Wake Tx queues
 675  * @param netdev network device
 676  */
 677 static inline void txqs_wake(struct net_device *netdev)
 678 {
 679         if (netif_is_multiqueue(netdev)) {
 680                 int i;
 681
 682                 for (i = 0; i < netdev->num_tx_queues; i++)
 683                         netif_wake_subqueue(netdev, i);
 684         } else {
 685                 netif_wake_queue(netdev);
 686         }
 687 }
 688
 689 /**
 690  * \brief Stop Tx queue
 691  * @param netdev network device
 692  */
 693 static void stop_txq(struct net_device *netdev)
 694 {
 695         txqs_stop(netdev);
 696 }
 697
 698 /**
 699  * \brief Start Tx queue
 700  * @param netdev network device
 701  */
 702 static void start_txq(struct net_device *netdev)
 703 {
 704         struct lio *lio = GET_LIO(netdev);
 705
 706         if (lio->linfo.link.s.status) {
 707                 txqs_start(netdev);
 708                 return;
 709         }
 710 }
 711
 712 /**
 713  * \brief Wake a queue
 714  * @param netdev network device
 715  * @param q which queue to wake
 716  */
 717 static inline void wake_q(struct net_device *netdev, int q)
 718 {
 719         if (netif_is_multiqueue(netdev))
 720                 netif_wake_subqueue(netdev, q);
 721         else
 722                 netif_wake_queue(netdev);
 723 }
 724
 725 /**
 726  * \brief Stop a queue
 727  * @param netdev network device
 728  * @param q which queue to stop
 729  */
 730 static inline void stop_q(struct net_device *netdev, int q)
 731 {
 732         if (netif_is_multiqueue(netdev))
 733                 netif_stop_subqueue(netdev, q);
 734         else
 735                 netif_stop_queue(netdev);
 736 }
 737
 738 /**
 739  * \brief Check Tx queue status, and take appropriate action
 740  * @param lio per-network private data
 741  * @returns 0 if full, number of queues woken up otherwise
 742  */
 743 static inline int check_txq_status(struct lio *lio)
 744 {
 745         int ret_val = 0;
 746
 747         if (netif_is_multiqueue(lio->netdev)) {
 748                 int numqs = lio->netdev->num_tx_queues;
 749                 int q, iq = 0;
 750
 751                 /* check each sub-queue state */
 752                 for (q = 0; q < numqs; q++) {
 753                         iq = lio->linfo.txpciq[q & (lio->linfo.num_txpciq - 1)];
 754                         if (octnet_iq_is_full(lio->oct_dev, iq))
 755                                 continue;
 756                         wake_q(lio->netdev, q);
 757                         ret_val++;
 758                 }
 759         } else {
 760                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 761                         return 0;
 762                 wake_q(lio->netdev, lio->txq);
 763                 ret_val = 1;
 764         }
 765         return ret_val;
 766 }
 767
 768 /**
 769  * Remove the node at the head of the list. The list would be empty at
 770  * the end of this call if there are no more nodes in the list.
 771  */
 772 static inline struct list_head *list_delete_head(struct list_head *root)
 773 {
 774         struct list_head *node;
 775
 776         if ((root->prev == root) && (root->next == root))
 777                 node = NULL;
 778         else
 779                 node = root->next;
 780
 781         if (node)
 782                 list_del(node);
 783
 784         return node;
 785 }
 786
 787 /**
 788  * \brief Delete gather list
 789  * @param lio per-network private data
 790  */
 791 static void delete_glist(struct lio *lio)
 792 {
 793         struct octnic_gather *g;
 794
 795         do {
 796                 g = (struct octnic_gather *)
 797                     list_delete_head(&lio->glist);
 798                 if (g) {
 799                         if (g->sg)
 800                                 kfree((void *)((unsigned long)g->sg -
 801                                                 g->adjust));
 802                         kfree(g);
 803                 }
 804         } while (g);
 805 }
 806
 807 /**
 808  * \brief Setup gather list
 809  * @param lio per-network private data
 810  */
 811 static int setup_glist(struct lio *lio)
 812 {
 813         int i;
 814         struct octnic_gather *g;
 815
 816         INIT_LIST_HEAD(&lio->glist);
 817
 818         for (i = 0; i < lio->tx_qsize; i++) {
 819                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 820                 if (!g)
 821                         break;
 822
 823                 g->sg_size =
 824                         ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
 825
 826                 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
 827                 if (!g->sg) {
 828                         kfree(g);
 829                         break;
 830                 }
 831
 832                 /* The gather component should be aligned on 64-bit boundary */
 833                 if (((unsigned long)g->sg) & 7) {
 834                         g->adjust = 8 - (((unsigned long)g->sg) & 7);
 835                         g->sg = (struct octeon_sg_entry *)
 836                                 ((unsigned long)g->sg + g->adjust);
 837                 }
 838                 list_add_tail(&g->list, &lio->glist);
 839         }
 840
 841         if (i == lio->tx_qsize)
 842                 return 0;
 843
 844         delete_glist(lio);
 845         return 1;
 846 }
 847
 848 /**
 849  * \brief Print link information
 850  * @param netdev network device
 851  */
 852 static void print_link_info(struct net_device *netdev)
 853 {
 854         struct lio *lio = GET_LIO(netdev);
 855
 856         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 857                 struct oct_link_info *linfo = &lio->linfo;
 858
 859                 if (linfo->link.s.status) {
 860                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 861                                    linfo->link.s.speed,
 862                                    (linfo->link.s.duplex) ? "Full" : "Half");
 863                 } else {
 864                         netif_info(lio, link, lio->netdev, "Link Down\n");
 865                 }
 866         }
 867 }
 868
 869 /**
 870  * \brief Update link status
 871  * @param netdev network device
 872  * @param ls link status structure
 873  *
 874  * Called on receipt of a link status response from the core application to
 875  * update each interface's link status.
 876  */
 877 static inline void update_link_status(struct net_device *netdev,
 878                                       union oct_link_status *ls)
 879 {
 880         struct lio *lio = GET_LIO(netdev);
 881
 882         if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) {
 883                 lio->linfo.link.u64 = ls->u64;
 884
 885                 print_link_info(netdev);
 886
 887                 if (lio->linfo.link.s.status) {
 888                         netif_carrier_on(netdev);
 889                         /* start_txq(netdev); */
 890                         txqs_wake(netdev);
 891                 } else {
 892                         netif_carrier_off(netdev);
 893                         stop_txq(netdev);
 894                 }
 895         }
 896 }
 897
 898 /**
 899  * \brief Droq packet processor sceduler
 900  * @param oct octeon device
 901  */
 902 static
 903 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 904 {
 905         struct octeon_device_priv *oct_priv =
 906                 (struct octeon_device_priv *)oct->priv;
 907         u64 oq_no;
 908         struct octeon_droq *droq;
 909
 910         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 911                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) {
 912                         if (!(oct->droq_intr & (1 << oq_no)))
 913                                 continue;
 914
 915                         droq = oct->droq[oq_no];
 916
 917                         if (droq->ops.poll_mode) {
 918                                 droq->ops.napi_fn(droq);
 919                                 oct_priv->napi_mask |= (1 << oq_no);
 920                         } else {
 921                                 tasklet_schedule(&oct_priv->droq_tasklet);
 922                         }
 923                 }
 924         }
 925 }
 926
 927 /**
 928  * \brief Interrupt handler for octeon
 929  * @param irq unused
 930  * @param dev octeon device
 931  */
 932 static
 933 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 934 {
 935         struct octeon_device *oct = (struct octeon_device *)dev;
 936         irqreturn_t ret;
 937
 938         /* Disable our interrupts for the duration of ISR */
 939         oct->fn_list.disable_interrupt(oct->chip);
 940
 941         ret = oct->fn_list.process_interrupt_regs(oct);
 942
 943         if (ret == IRQ_HANDLED)
 944                 liquidio_schedule_droq_pkt_handlers(oct);
 945
 946         /* Re-enable our interrupts  */
 947         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
 948                 oct->fn_list.enable_interrupt(oct->chip);
 949
 950         return ret;
 951 }
 952
 953 /**
 954  * \brief Setup interrupt for octeon device
 955  * @param oct octeon device
 956  *
 957  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
 958  */
 959 static int octeon_setup_interrupt(struct octeon_device *oct)
 960 {
 961         int irqret, err;
 962
 963         err = pci_enable_msi(oct->pci_dev);
 964         if (err)
 965                 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
 966                          err);
 967         else
 968                 oct->flags |= LIO_FLAG_MSI_ENABLED;
 969
 970         irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
 971                              IRQF_SHARED, "octeon", oct);
 972         if (irqret) {
 973                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
 974                         pci_disable_msi(oct->pci_dev);
 975                 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
 976                         irqret);
 977                 return 1;
 978         }
 979
 980         return 0;
 981 }
 982
 983 /**
 984  * \brief PCI probe handler
 985  * @param pdev PCI device structure
 986  * @param ent unused
 987  */
 988 static int liquidio_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 989 {
 990         struct octeon_device *oct_dev = NULL;
 991         struct handshake *hs;
 992
 993         oct_dev = octeon_allocate_device(pdev->device,
 994                                          sizeof(struct octeon_device_priv));
 995         if (!oct_dev) {
 996                 dev_err(&pdev->dev, "Unable to allocate device\n");
 997                 return -ENOMEM;
 998         }
 999
1000         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1001                  (u32)pdev->vendor, (u32)pdev->device);
1002
1003         /* Assign octeon_device for this device to the private data area. */
1004         pci_set_drvdata(pdev, oct_dev);
1005
1006         /* set linux specific device pointer */
1007         oct_dev->pci_dev = (void *)pdev;
1008
1009         hs = &handshake[oct_dev->octeon_id];
1010         init_completion(&hs->init);
1011         init_completion(&hs->started);
1012         hs->pci_dev = pdev;
1013
1014         if (oct_dev->octeon_id == 0)
1015                 /* first LiquidIO NIC is detected */
1016                 complete(&first_stage);
1017
1018         if (octeon_device_init(oct_dev)) {
1019                 liquidio_remove(pdev);
1020                 return -ENOMEM;
1021         }
1022
1023         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1024
1025         return 0;
1026 }
1027
1028 /**
1029  *\brief Destroy resources associated with octeon device
1030  * @param pdev PCI device structure
1031  * @param ent unused
1032  */
1033 static void octeon_destroy_resources(struct octeon_device *oct)
1034 {
1035         int i;
1036         struct octeon_device_priv *oct_priv =
1037                 (struct octeon_device_priv *)oct->priv;
1038
1039         struct handshake *hs;
1040
1041         switch (atomic_read(&oct->status)) {
1042         case OCT_DEV_RUNNING:
1043         case OCT_DEV_CORE_OK:
1044
1045                 /* No more instructions will be forwarded. */
1046                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1047
1048                 oct->app_mode = CVM_DRV_INVALID_APP;
1049                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1050                         lio_get_state_string(&oct->status));
1051
1052                 schedule_timeout_uninterruptible(HZ / 10);
1053
1054                 /* fallthrough */
1055         case OCT_DEV_HOST_OK:
1056
1057                 /* fallthrough */
1058         case OCT_DEV_CONSOLE_INIT_DONE:
1059                 /* Remove any consoles */
1060                 octeon_remove_consoles(oct);
1061
1062                 /* fallthrough */
1063         case OCT_DEV_IO_QUEUES_DONE:
1064                 if (wait_for_pending_requests(oct))
1065                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1066
1067                 if (lio_wait_for_instr_fetch(oct))
1068                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1069
1070                 /* Disable the input and output queues now. No more packets will
1071                  * arrive from Octeon, but we should wait for all packet
1072                  * processing to finish.
1073                  */
1074                 oct->fn_list.disable_io_queues(oct);
1075
1076                 if (lio_wait_for_oq_pkts(oct))
1077                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1078
1079                 /* Disable interrupts  */
1080                 oct->fn_list.disable_interrupt(oct->chip);
1081
1082                 /* Release the interrupt line */
1083                 free_irq(oct->pci_dev->irq, oct);
1084
1085                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1086                         pci_disable_msi(oct->pci_dev);
1087
1088                 /* Soft reset the octeon device before exiting */
1089                 oct->fn_list.soft_reset(oct);
1090
1091                 /* Disable the device, releasing the PCI INT */
1092                 pci_disable_device(oct->pci_dev);
1093
1094                 /* fallthrough */
1095         case OCT_DEV_IN_RESET:
1096         case OCT_DEV_DROQ_INIT_DONE:
1097                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1098                 mdelay(100);
1099                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
1100                         if (!(oct->io_qmask.oq & (1UL << i)))
1101                                 continue;
1102                         octeon_delete_droq(oct, i);
1103                 }
1104
1105                 /* Force any pending handshakes to complete */
1106                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1107                         hs = &handshake[i];
1108
1109                         if (hs->pci_dev) {
1110                                 handshake[oct->octeon_id].init_ok = 0;
1111                                 complete(&handshake[oct->octeon_id].init);
1112                                 handshake[oct->octeon_id].started_ok = 0;
1113                                 complete(&handshake[oct->octeon_id].started);
1114                         }
1115                 }
1116
1117                 /* fallthrough */
1118         case OCT_DEV_RESP_LIST_INIT_DONE:
1119                 octeon_delete_response_list(oct);
1120
1121                 /* fallthrough */
1122         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1123                 octeon_free_sc_buffer_pool(oct);
1124
1125                 /* fallthrough */
1126         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1127                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
1128                         if (!(oct->io_qmask.iq & (1UL << i)))
1129                                 continue;
1130                         octeon_delete_instr_queue(oct, i);
1131                 }
1132
1133                 /* fallthrough */
1134         case OCT_DEV_DISPATCH_INIT_DONE:
1135                 octeon_delete_dispatch_list(oct);
1136                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1137
1138                 /* fallthrough */
1139         case OCT_DEV_PCI_MAP_DONE:
1140                 octeon_unmap_pci_barx(oct, 0);
1141                 octeon_unmap_pci_barx(oct, 1);
1142
1143                 /* fallthrough */
1144         case OCT_DEV_BEGIN_STATE:
1145                 /* Nothing to be done here either */
1146                 break;
1147         }                       /* end switch(oct->status) */
1148
1149         tasklet_kill(&oct_priv->droq_tasklet);
1150 }
1151
1152 /**
1153  * \brief Send Rx control command
1154  * @param lio per-network private data
1155  * @param start_stop whether to start or stop
1156  */
1157 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1158 {
1159         struct octnic_ctrl_pkt nctrl;
1160         struct octnic_ctrl_params nparams;
1161
1162         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1163
1164         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
1165         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
1166         nctrl.ncmd.s.param2 = start_stop;
1167         nctrl.netpndev = (u64)lio->netdev;
1168
1169         nparams.resp_order = OCTEON_RESP_NORESPONSE;
1170
1171         if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams) < 0)
1172                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1173 }
1174
1175 /**
1176  * \brief Destroy NIC device interface
1177  * @param oct octeon device
1178  * @param ifidx which interface to destroy
1179  *
1180  * Cleanup associated with each interface for an Octeon device  when NIC
1181  * module is being unloaded or if initialization fails during load.
1182  */
1183 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1184 {
1185         struct net_device *netdev = oct->props[ifidx].netdev;
1186         struct lio *lio;
1187
1188         if (!netdev) {
1189                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1190                         __func__, ifidx);
1191                 return;
1192         }
1193
1194         lio = GET_LIO(netdev);
1195
1196         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1197
1198         send_rx_ctrl_cmd(lio, 0);
1199
1200         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1201                 txqs_stop(netdev);
1202
1203         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1204                 unregister_netdev(netdev);
1205
1206         delete_glist(lio);
1207
1208         free_netdev(netdev);
1209
1210         oct->props[ifidx].netdev = NULL;
1211 }
1212
1213 /**
1214  * \brief Stop complete NIC functionality
1215  * @param oct octeon device
1216  */
1217 static int liquidio_stop_nic_module(struct octeon_device *oct)
1218 {
1219         int i, j;
1220         struct lio *lio;
1221
1222         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1223         if (!oct->ifcount) {
1224                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1225                 return 1;
1226         }
1227
1228         for (i = 0; i < oct->ifcount; i++) {
1229                 lio = GET_LIO(oct->props[i].netdev);
1230                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1231                         octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j]);
1232         }
1233
1234         for (i = 0; i < oct->ifcount; i++)
1235                 liquidio_destroy_nic_device(oct, i);
1236
1237         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1238         return 0;
1239 }
1240
1241 /**
1242  * \brief Cleans up resources at unload time
1243  * @param pdev PCI device structure
1244  */
1245 static void liquidio_remove(struct pci_dev *pdev)
1246 {
1247         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1248
1249         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1250
1251         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1252                 liquidio_stop_nic_module(oct_dev);
1253
1254         /* Reset the octeon device and cleanup all memory allocated for
1255          * the octeon device by driver.
1256          */
1257         octeon_destroy_resources(oct_dev);
1258
1259         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1260
1261         /* This octeon device has been removed. Update the global
1262          * data structure to reflect this. Free the device structure.
1263          */
1264         octeon_free_device_mem(oct_dev);
1265 }
1266
1267 /**
1268  * \brief Identify the Octeon device and to map the BAR address space
1269  * @param oct octeon device
1270  */
1271 static int octeon_chip_specific_setup(struct octeon_device *oct)
1272 {
1273         u32 dev_id, rev_id;
1274         int ret = 1;
1275
1276         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1277         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1278         oct->rev_id = rev_id & 0xff;
1279
1280         switch (dev_id) {
1281         case OCTEON_CN68XX_PCIID:
1282                 oct->chip_id = OCTEON_CN68XX;
1283                 ret = lio_setup_cn68xx_octeon_device(oct);
1284                 break;
1285
1286         case OCTEON_CN66XX_PCIID:
1287                 oct->chip_id = OCTEON_CN66XX;
1288                 ret = lio_setup_cn66xx_octeon_device(oct);
1289                 break;
1290         default:
1291                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1292                         dev_id);
1293         }
1294
1295         if (!ret)
1296                 dev_info(&oct->pci_dev->dev, "CN68XX PASS%d.%d %s\n",
1297                          OCTEON_MAJOR_REV(oct),
1298                          OCTEON_MINOR_REV(oct),
1299                          octeon_get_conf(oct)->card_name);
1300
1301         return ret;
1302 }
1303
1304 /**
1305  * \brief PCI initialization for each Octeon device.
1306  * @param oct octeon device
1307  */
1308 static int octeon_pci_os_setup(struct octeon_device *oct)
1309 {
1310         /* setup PCI stuff first */
1311         if (pci_enable_device(oct->pci_dev)) {
1312                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1313                 return 1;
1314         }
1315
1316         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1317                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1318                 return 1;
1319         }
1320
1321         /* Enable PCI DMA Master. */
1322         pci_set_master(oct->pci_dev);
1323
1324         return 0;
1325 }
1326
1327 /**
1328  * \brief Check Tx queue state for a given network buffer
1329  * @param lio per-network private data
1330  * @param skb network buffer
1331  */
1332 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1333 {
1334         int q = 0, iq = 0;
1335
1336         if (netif_is_multiqueue(lio->netdev)) {
1337                 q = skb->queue_mapping;
1338                 iq = lio->linfo.txpciq[(q & (lio->linfo.num_txpciq - 1))];
1339         } else {
1340                 iq = lio->txq;
1341         }
1342
1343         if (octnet_iq_is_full(lio->oct_dev, iq))
1344                 return 0;
1345         wake_q(lio->netdev, q);
1346         return 1;
1347 }
1348
1349 /**
1350  * \brief Unmap and free network buffer
1351  * @param buf buffer
1352  */
1353 static void free_netbuf(void *buf)
1354 {
1355         struct sk_buff *skb;
1356         struct octnet_buf_free_info *finfo;
1357         struct lio *lio;
1358
1359         finfo = (struct octnet_buf_free_info *)buf;
1360         skb = finfo->skb;
1361         lio = finfo->lio;
1362
1363         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1364                          DMA_TO_DEVICE);
1365
1366         check_txq_state(lio, skb);
1367
1368         recv_buffer_free((struct sk_buff *)skb);
1369 }
1370
1371 /**
1372  * \brief Unmap and free gather buffer
1373  * @param buf buffer
1374  */
1375 static void free_netsgbuf(void *buf)
1376 {
1377         struct octnet_buf_free_info *finfo;
1378         struct sk_buff *skb;
1379         struct lio *lio;
1380         struct octnic_gather *g;
1381         int i, frags;
1382
1383         finfo = (struct octnet_buf_free_info *)buf;
1384         skb = finfo->skb;
1385         lio = finfo->lio;
1386         g = finfo->g;
1387         frags = skb_shinfo(skb)->nr_frags;
1388
1389         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1390                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1391                          DMA_TO_DEVICE);
1392
1393         i = 1;
1394         while (frags--) {
1395                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1396
1397                 pci_unmap_page((lio->oct_dev)->pci_dev,
1398                                g->sg[(i >> 2)].ptr[(i & 3)],
1399                                frag->size, DMA_TO_DEVICE);
1400                 i++;
1401         }
1402
1403         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1404                          finfo->dptr, g->sg_size,
1405                          DMA_TO_DEVICE);
1406
1407         spin_lock(&lio->lock);
1408         list_add_tail(&g->list, &lio->glist);
1409         spin_unlock(&lio->lock);
1410
1411         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1412
1413         recv_buffer_free((struct sk_buff *)skb);
1414 }
1415
1416 /**
1417  * \brief Unmap and free gather buffer with response
1418  * @param buf buffer
1419  */
1420 static void free_netsgbuf_with_resp(void *buf)
1421 {
1422         struct octeon_soft_command *sc;
1423         struct octnet_buf_free_info *finfo;
1424         struct sk_buff *skb;
1425         struct lio *lio;
1426         struct octnic_gather *g;
1427         int i, frags;
1428
1429         sc = (struct octeon_soft_command *)buf;
1430         skb = (struct sk_buff *)sc->callback_arg;
1431         finfo = (struct octnet_buf_free_info *)&skb->cb;
1432
1433         lio = finfo->lio;
1434         g = finfo->g;
1435         frags = skb_shinfo(skb)->nr_frags;
1436
1437         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1438                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1439                          DMA_TO_DEVICE);
1440
1441         i = 1;
1442         while (frags--) {
1443                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1444
1445                 pci_unmap_page((lio->oct_dev)->pci_dev,
1446                                g->sg[(i >> 2)].ptr[(i & 3)],
1447                                frag->size, DMA_TO_DEVICE);
1448                 i++;
1449         }
1450
1451         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1452                          finfo->dptr, g->sg_size,
1453                          DMA_TO_DEVICE);
1454
1455         spin_lock(&lio->lock);
1456         list_add_tail(&g->list, &lio->glist);
1457         spin_unlock(&lio->lock);
1458
1459         /* Don't free the skb yet */
1460
1461         check_txq_state(lio, skb);
1462 }
1463
1464 /**
1465  * \brief Adjust ptp frequency
1466  * @param ptp PTP clock info
1467  * @param ppb how much to adjust by, in parts-per-billion
1468  */
1469 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1470 {
1471         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1472         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1473         u64 comp, delta;
1474         unsigned long flags;
1475         bool neg_adj = false;
1476
1477         if (ppb < 0) {
1478                 neg_adj = true;
1479                 ppb = -ppb;
1480         }
1481
1482         /* The hardware adds the clock compensation value to the
1483          * PTP clock on every coprocessor clock cycle, so we
1484          * compute the delta in terms of coprocessor clocks.
1485          */
1486         delta = (u64)ppb << 32;
1487         do_div(delta, oct->coproc_clock_rate);
1488
1489         spin_lock_irqsave(&lio->ptp_lock, flags);
1490         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1491         if (neg_adj)
1492                 comp -= delta;
1493         else
1494                 comp += delta;
1495         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1496         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1497
1498         return 0;
1499 }
1500
1501 /**
1502  * \brief Adjust ptp time
1503  * @param ptp PTP clock info
1504  * @param delta how much to adjust by, in nanosecs
1505  */
1506 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1507 {
1508         unsigned long flags;
1509         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1510
1511         spin_lock_irqsave(&lio->ptp_lock, flags);
1512         lio->ptp_adjust += delta;
1513         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1514
1515         return 0;
1516 }
1517
1518 /**
1519  * \brief Get hardware clock time, including any adjustment
1520  * @param ptp PTP clock info
1521  * @param ts timespec
1522  */
1523 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1524                                 struct timespec64 *ts)
1525 {
1526         u64 ns;
1527         u32 remainder;
1528         unsigned long flags;
1529         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1530         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1531
1532         spin_lock_irqsave(&lio->ptp_lock, flags);
1533         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1534         ns += lio->ptp_adjust;
1535         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1536
1537         ts->tv_sec = div_u64_rem(ns, 1000000000ULL, &remainder);
1538         ts->tv_nsec = remainder;
1539
1540         return 0;
1541 }
1542
1543 /**
1544  * \brief Set hardware clock time. Reset adjustment
1545  * @param ptp PTP clock info
1546  * @param ts timespec
1547  */
1548 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1549                                 const struct timespec64 *ts)
1550 {
1551         u64 ns;
1552         unsigned long flags;
1553         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1554         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1555
1556         ns = timespec_to_ns(ts);
1557
1558         spin_lock_irqsave(&lio->ptp_lock, flags);
1559         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1560         lio->ptp_adjust = 0;
1561         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1562
1563         return 0;
1564 }
1565
1566 /**
1567  * \brief Check if PTP is enabled
1568  * @param ptp PTP clock info
1569  * @param rq request
1570  * @param on is it on
1571  */
1572 static int liquidio_ptp_enable(struct ptp_clock_info *ptp,
1573                                struct ptp_clock_request *rq, int on)
1574 {
1575         return -EOPNOTSUPP;
1576 }
1577
1578 /**
1579  * \brief Open PTP clock source
1580  * @param netdev network device
1581  */
1582 static void oct_ptp_open(struct net_device *netdev)
1583 {
1584         struct lio *lio = GET_LIO(netdev);
1585         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1586
1587         spin_lock_init(&lio->ptp_lock);
1588
1589         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1590         lio->ptp_info.owner = THIS_MODULE;
1591         lio->ptp_info.max_adj = 250000000;
1592         lio->ptp_info.n_alarm = 0;
1593         lio->ptp_info.n_ext_ts = 0;
1594         lio->ptp_info.n_per_out = 0;
1595         lio->ptp_info.pps = 0;
1596         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1597         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1598         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1599         lio->ptp_info.settime64 = liquidio_ptp_settime;
1600         lio->ptp_info.enable = liquidio_ptp_enable;
1601
1602         lio->ptp_adjust = 0;
1603
1604         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1605                                              &oct->pci_dev->dev);
1606
1607         if (IS_ERR(lio->ptp_clock))
1608                 lio->ptp_clock = NULL;
1609 }
1610
1611 /**
1612  * \brief Init PTP clock
1613  * @param oct octeon device
1614  */
1615 static void liquidio_ptp_init(struct octeon_device *oct)
1616 {
1617         u64 clock_comp, cfg;
1618
1619         clock_comp = (u64)NSEC_PER_SEC << 32;
1620         do_div(clock_comp, oct->coproc_clock_rate);
1621         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1622
1623         /* Enable */
1624         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1625         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1626 }
1627
1628 /**
1629  * \brief Load firmware to device
1630  * @param oct octeon device
1631  *
1632  * Maps device to firmware filename, requests firmware, and downloads it
1633  */
1634 static int load_firmware(struct octeon_device *oct)
1635 {
1636         int ret = 0;
1637         const struct firmware *fw;
1638         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1639         char *tmp_fw_type;
1640
1641         if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1642                     sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
1643                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
1644                 return ret;
1645         }
1646
1647         if (fw_type[0] == '\0')
1648                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1649         else
1650                 tmp_fw_type = fw_type;
1651
1652         sprintf(fw_name, "/*(DEBLOBBED)*/", LIO_FW_DIR, LIO_FW_BASE_NAME,
1653                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1654                 LIO_FW_NAME_SUFFIX);
1655
1656         ret = reject_firmware(&fw, fw_name, &oct->pci_dev->dev);
1657         if (ret) {
1658                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
1659                         fw_name);
1660                 return ret;
1661         }
1662
1663         ret = octeon_download_firmware(oct, fw->data, fw->size);
1664
1665         release_firmware(fw);
1666
1667         return ret;
1668 }
1669
1670 /**
1671  * \brief Setup output queue
1672  * @param oct octeon device
1673  * @param q_no which queue
1674  * @param num_descs how many descriptors
1675  * @param desc_size size of each descriptor
1676  * @param app_ctx application context
1677  */
1678 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
1679                              int desc_size, void *app_ctx)
1680 {
1681         int ret_val = 0;
1682
1683         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
1684         /* droq creation and local register settings. */
1685         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
1686         if (ret_val == -1)
1687                 return ret_val;
1688
1689         if (ret_val == 1) {
1690                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
1691                 return 0;
1692         }
1693         /* tasklet creation for the droq */
1694
1695         /* Enable the droq queues */
1696         octeon_set_droq_pkt_op(oct, q_no, 1);
1697
1698         /* Send Credit for Octeon Output queues. Credits are always
1699          * sent after the output queue is enabled.
1700          */
1701         writel(oct->droq[q_no]->max_count,
1702                oct->droq[q_no]->pkts_credit_reg);
1703
1704         return ret_val;
1705 }
1706
1707 /**
1708  * \brief Callback for getting interface configuration
1709  * @param status status of request
1710  * @param buf pointer to resp structure
1711  */
1712 static void if_cfg_callback(struct octeon_device *oct,
1713                             u32 status,
1714                             void *buf)
1715 {
1716         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1717         struct liquidio_if_cfg_resp *resp;
1718         struct liquidio_if_cfg_context *ctx;
1719
1720         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1721         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1722
1723         oct = lio_get_device(ctx->octeon_id);
1724         if (resp->status)
1725                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
1726                         CVM_CAST64(resp->status));
1727         ACCESS_ONCE(ctx->cond) = 1;
1728
1729         /* This barrier is required to be sure that the response has been
1730          * written fully before waking up the handler
1731          */
1732         wmb();
1733
1734         wake_up_interruptible(&ctx->wc);
1735 }
1736
1737 /**
1738  * \brief Select queue based on hash
1739  * @param dev Net device
1740  * @param skb sk_buff structure
1741  * @returns selected queue number
1742  */
1743 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
1744                     void *accel_priv, select_queue_fallback_t fallback)
1745 {
1746         int qindex;
1747         struct lio *lio;
1748
1749         lio = GET_LIO(dev);
1750         /* select queue on chosen queue_mapping or core */
1751         qindex = skb_rx_queue_recorded(skb) ?
1752                  skb_get_rx_queue(skb) : smp_processor_id();
1753         return (u16)(qindex & (lio->linfo.num_txpciq - 1));
1754 }
1755
1756 /** Routine to push packets arriving on Octeon interface upto network layer.
1757  * @param oct_id   - octeon device id.
1758  * @param skbuff   - skbuff struct to be passed to network layer.
1759  * @param len      - size of total data received.
1760  * @param rh       - Control header associated with the packet
1761  * @param param    - additional control data with the packet
1762  */
1763 static void
1764 liquidio_push_packet(u32 octeon_id,
1765                      void *skbuff,
1766                      u32 len,
1767                      union octeon_rh *rh,
1768                      void *param)
1769 {
1770         struct napi_struct *napi = param;
1771         struct octeon_device *oct = lio_get_device(octeon_id);
1772         struct sk_buff *skb = (struct sk_buff *)skbuff;
1773         struct skb_shared_hwtstamps *shhwtstamps;
1774         u64 ns;
1775         struct net_device *netdev =
1776                 (struct net_device *)oct->props[rh->r_dh.link].netdev;
1777         struct octeon_droq *droq = container_of(param, struct octeon_droq,
1778                                                 napi);
1779         if (netdev) {
1780                 int packet_was_received;
1781                 struct lio *lio = GET_LIO(netdev);
1782
1783                 /* Do not proceed if the interface is not in RUNNING state. */
1784                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1785                         recv_buffer_free(skb);
1786                         droq->stats.rx_dropped++;
1787                         return;
1788                 }
1789
1790                 skb->dev = netdev;
1791
1792                 if (rh->r_dh.has_hwtstamp) {
1793                         /* timestamp is included from the hardware at the
1794                          * beginning of the packet.
1795                          */
1796                         if (ifstate_check(lio,
1797                                           LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
1798                                 /* Nanoseconds are in the first 64-bits
1799                                  * of the packet.
1800                                  */
1801                                 memcpy(&ns, (skb->data), sizeof(ns));
1802                                 shhwtstamps = skb_hwtstamps(skb);
1803                                 shhwtstamps->hwtstamp =
1804                                         ns_to_ktime(ns + lio->ptp_adjust);
1805                         }
1806                         skb_pull(skb, sizeof(ns));
1807                 }
1808
1809                 skb->protocol = eth_type_trans(skb, skb->dev);
1810
1811                 if ((netdev->features & NETIF_F_RXCSUM) &&
1812                     (rh->r_dh.csum_verified == CNNIC_CSUM_VERIFIED))
1813                         /* checksum has already been verified */
1814                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1815                 else
1816                         skb->ip_summed = CHECKSUM_NONE;
1817
1818                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
1819
1820                 if (packet_was_received) {
1821                         droq->stats.rx_bytes_received += len;
1822                         droq->stats.rx_pkts_received++;
1823                         netdev->last_rx = jiffies;
1824                 } else {
1825                         droq->stats.rx_dropped++;
1826                         netif_info(lio, rx_err, lio->netdev,
1827                                    "droq:%d  error rx_dropped:%llu\n",
1828                                    droq->q_no, droq->stats.rx_dropped);
1829                 }
1830
1831         } else {
1832                 recv_buffer_free(skb);
1833         }
1834 }
1835
1836 /**
1837  * \brief wrapper for calling napi_schedule
1838  * @param param parameters to pass to napi_schedule
1839  *
1840  * Used when scheduling on different CPUs
1841  */
1842 static void napi_schedule_wrapper(void *param)
1843 {
1844         struct napi_struct *napi = param;
1845
1846         napi_schedule(napi);
1847 }
1848
1849 /**
1850  * \brief callback when receive interrupt occurs and we are in NAPI mode
1851  * @param arg pointer to octeon output queue
1852  */
1853 static void liquidio_napi_drv_callback(void *arg)
1854 {
1855         struct octeon_droq *droq = arg;
1856         int this_cpu = smp_processor_id();
1857
1858         if (droq->cpu_id == this_cpu) {
1859                 napi_schedule(&droq->napi);
1860         } else {
1861                 struct call_single_data *csd = &droq->csd;
1862
1863                 csd->func = napi_schedule_wrapper;
1864                 csd->info = &droq->napi;
1865                 csd->flags = 0;
1866
1867                 smp_call_function_single_async(droq->cpu_id, csd);
1868         }
1869 }
1870
1871 /**
1872  * \brief Main NAPI poll function
1873  * @param droq octeon output queue
1874  * @param budget maximum number of items to process
1875  */
1876 static int liquidio_napi_do_rx(struct octeon_droq *droq, int budget)
1877 {
1878         int work_done;
1879         struct lio *lio = GET_LIO(droq->napi.dev);
1880         struct octeon_device *oct = lio->oct_dev;
1881
1882         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
1883                                                  POLL_EVENT_PROCESS_PKTS,
1884                                                  budget);
1885         if (work_done < 0) {
1886                 netif_info(lio, rx_err, lio->netdev,
1887                            "Receive work_done < 0, rxq:%d\n", droq->q_no);
1888                 goto octnet_napi_finish;
1889         }
1890
1891         if (work_done > budget)
1892                 dev_err(&oct->pci_dev->dev, ">>>> %s work_done: %d budget: %d\n",
1893                         __func__, work_done, budget);
1894
1895         return work_done;
1896
1897 octnet_napi_finish:
1898         napi_complete(&droq->napi);
1899         octeon_process_droq_poll_cmd(oct, droq->q_no, POLL_EVENT_ENABLE_INTR,
1900                                      0);
1901         return 0;
1902 }
1903
1904 /**
1905  * \brief Entry point for NAPI polling
1906  * @param napi NAPI structure
1907  * @param budget maximum number of items to process
1908  */
1909 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
1910 {
1911         struct octeon_droq *droq;
1912         int work_done;
1913
1914         droq = container_of(napi, struct octeon_droq, napi);
1915
1916         work_done = liquidio_napi_do_rx(droq, budget);
1917
1918         if (work_done < budget) {
1919                 napi_complete(napi);
1920                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
1921                                              POLL_EVENT_ENABLE_INTR, 0);
1922                 return 0;
1923         }
1924
1925         return work_done;
1926 }
1927
1928 /**
1929  * \brief Setup input and output queues
1930  * @param octeon_dev octeon device
1931  * @param net_device Net device
1932  *
1933  * Note: Queues are with respect to the octeon device. Thus
1934  * an input queue is for egress packets, and output queues
1935  * are for ingress packets.
1936  */
1937 static inline int setup_io_queues(struct octeon_device *octeon_dev,
1938                                   struct net_device *net_device)
1939 {
1940         static int first_time = 1;
1941         static struct octeon_droq_ops droq_ops;
1942         static int cpu_id;
1943         static int cpu_id_modulus;
1944         struct octeon_droq *droq;
1945         struct napi_struct *napi;
1946         int q, q_no, retval = 0;
1947         struct lio *lio;
1948         int num_tx_descs;
1949
1950         lio = GET_LIO(net_device);
1951         if (first_time) {
1952                 first_time = 0;
1953                 memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
1954
1955                 droq_ops.fptr = liquidio_push_packet;
1956
1957                 droq_ops.poll_mode = 1;
1958                 droq_ops.napi_fn = liquidio_napi_drv_callback;
1959                 cpu_id = 0;
1960                 cpu_id_modulus = num_present_cpus();
1961         }
1962
1963         /* set up DROQs. */
1964         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
1965                 q_no = lio->linfo.rxpciq[q];
1966
1967                 retval = octeon_setup_droq(octeon_dev, q_no,
1968                                            CFG_GET_NUM_RX_DESCS_NIC_IF
1969                                                    (octeon_get_conf(octeon_dev),
1970                                                    lio->ifidx),
1971                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
1972                                                    (octeon_get_conf(octeon_dev),
1973                                                    lio->ifidx), NULL);
1974                 if (retval) {
1975                         dev_err(&octeon_dev->pci_dev->dev,
1976                                 " %s : Runtime DROQ(RxQ) creation failed.\n",
1977                                 __func__);
1978                         return 1;
1979                 }
1980
1981                 droq = octeon_dev->droq[q_no];
1982                 napi = &droq->napi;
1983                 netif_napi_add(net_device, napi, liquidio_napi_poll, 64);
1984
1985                 /* designate a CPU for this droq */
1986                 droq->cpu_id = cpu_id;
1987                 cpu_id++;
1988                 if (cpu_id >= cpu_id_modulus)
1989                         cpu_id = 0;
1990
1991                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
1992         }
1993
1994         /* set up IQs. */
1995         for (q = 0; q < lio->linfo.num_txpciq; q++) {
1996                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
1997                                                            (octeon_dev),
1998                                                            lio->ifidx);
1999                 retval = octeon_setup_iq(octeon_dev, lio->linfo.txpciq[q],
2000                                          num_tx_descs,
2001                                          netdev_get_tx_queue(net_device, q));
2002                 if (retval) {
2003                         dev_err(&octeon_dev->pci_dev->dev,
2004                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2005                                 __func__);
2006                         return 1;
2007                 }
2008         }
2009
2010         return 0;
2011 }
2012
2013 /**
2014  * \brief Poll routine for checking transmit queue status
2015  * @param work work_struct data structure
2016  */
2017 static void octnet_poll_check_txq_status(struct work_struct *work)
2018 {
2019         struct cavium_wk *wk = (struct cavium_wk *)work;
2020         struct lio *lio = (struct lio *)wk->ctxptr;
2021
2022         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2023                 return;
2024
2025         check_txq_status(lio);
2026         queue_delayed_work(lio->txq_status_wq.wq,
2027                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2028 }
2029
2030 /**
2031  * \brief Sets up the txq poll check
2032  * @param netdev network device
2033  */
2034 static inline void setup_tx_poll_fn(struct net_device *netdev)
2035 {
2036         struct lio *lio = GET_LIO(netdev);
2037         struct octeon_device *oct = lio->oct_dev;
2038
2039         lio->txq_status_wq.wq = create_workqueue("txq-status");
2040         if (!lio->txq_status_wq.wq) {
2041                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2042                 return;
2043         }
2044         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2045                           octnet_poll_check_txq_status);
2046         lio->txq_status_wq.wk.ctxptr = lio;
2047         queue_delayed_work(lio->txq_status_wq.wq,
2048                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2049 }
2050
2051 /**
2052  * \brief Net device open for LiquidIO
2053  * @param netdev network device
2054  */
2055 static int liquidio_open(struct net_device *netdev)
2056 {
2057         struct lio *lio = GET_LIO(netdev);
2058         struct octeon_device *oct = lio->oct_dev;
2059         struct napi_struct *napi, *n;
2060
2061         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2062                 napi_enable(napi);
2063
2064         oct_ptp_open(netdev);
2065
2066         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2067         setup_tx_poll_fn(netdev);
2068         start_txq(netdev);
2069
2070         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2071         try_module_get(THIS_MODULE);
2072
2073         /* tell Octeon to start forwarding packets to host */
2074         send_rx_ctrl_cmd(lio, 1);
2075
2076         /* Ready for link status updates */
2077         lio->intf_open = 1;
2078
2079         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2080                  netdev->name);
2081
2082         return 0;
2083 }
2084
2085 /**
2086  * \brief Net device stop for LiquidIO
2087  * @param netdev network device
2088  */
2089 static int liquidio_stop(struct net_device *netdev)
2090 {
2091         struct napi_struct *napi, *n;
2092         struct lio *lio = GET_LIO(netdev);
2093         struct octeon_device *oct = lio->oct_dev;
2094
2095         netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
2096         /* Inform that netif carrier is down */
2097         lio->intf_open = 0;
2098         lio->linfo.link.s.status = 0;
2099
2100         netif_carrier_off(netdev);
2101
2102         /* tell Octeon to stop forwarding packets to host */
2103         send_rx_ctrl_cmd(lio, 0);
2104
2105         cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2106         flush_workqueue(lio->txq_status_wq.wq);
2107         destroy_workqueue(lio->txq_status_wq.wq);
2108
2109         if (lio->ptp_clock) {
2110                 ptp_clock_unregister(lio->ptp_clock);
2111                 lio->ptp_clock = NULL;
2112         }
2113
2114         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2115
2116         /* This is a hack that allows DHCP to continue working. */
2117         set_bit(__LINK_STATE_START, &lio->netdev->state);
2118
2119         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2120                 napi_disable(napi);
2121
2122         txqs_stop(netdev);
2123
2124         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2125         module_put(THIS_MODULE);
2126
2127         return 0;
2128 }
2129
2130 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
2131 {
2132         struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr;
2133         struct net_device *netdev = (struct net_device *)nctrl->netpndev;
2134         struct lio *lio = GET_LIO(netdev);
2135         struct octeon_device *oct = lio->oct_dev;
2136
2137         switch (nctrl->ncmd.s.cmd) {
2138         case OCTNET_CMD_CHANGE_DEVFLAGS:
2139         case OCTNET_CMD_SET_MULTI_LIST:
2140                 break;
2141
2142         case OCTNET_CMD_CHANGE_MACADDR:
2143                 /* If command is successful, change the MACADDR. */
2144                 netif_info(lio, probe, lio->netdev, " MACAddr changed to 0x%llx\n",
2145                            CVM_CAST64(nctrl->udd[0]));
2146                 dev_info(&oct->pci_dev->dev, "%s MACAddr changed to 0x%llx\n",
2147                          netdev->name, CVM_CAST64(nctrl->udd[0]));
2148                 memcpy(netdev->dev_addr, ((u8 *)&nctrl->udd[0]) + 2, ETH_ALEN);
2149                 break;
2150
2151         case OCTNET_CMD_CHANGE_MTU:
2152                 /* If command is successful, change the MTU. */
2153                 netif_info(lio, probe, lio->netdev, " MTU Changed from %d to %d\n",
2154                            netdev->mtu, nctrl->ncmd.s.param2);
2155                 dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
2156                          netdev->name, netdev->mtu,
2157                          nctrl->ncmd.s.param2);
2158                 netdev->mtu = nctrl->ncmd.s.param2;
2159                 break;
2160
2161         case OCTNET_CMD_GPIO_ACCESS:
2162                 netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
2163
2164                 break;
2165
2166         case OCTNET_CMD_LRO_ENABLE:
2167                 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
2168                 break;
2169
2170         case OCTNET_CMD_LRO_DISABLE:
2171                 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
2172                          netdev->name);
2173                 break;
2174
2175         case OCTNET_CMD_VERBOSE_ENABLE:
2176                 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
2177                 break;
2178
2179         case OCTNET_CMD_VERBOSE_DISABLE:
2180                 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
2181                          netdev->name);
2182                 break;
2183
2184         case OCTNET_CMD_SET_SETTINGS:
2185                 dev_info(&oct->pci_dev->dev, "%s settings changed\n",
2186                          netdev->name);
2187
2188                 break;
2189
2190         default:
2191                 dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
2192                         nctrl->ncmd.s.cmd);
2193         }
2194 }
2195
2196 /**
2197  * \brief Converts a mask based on net device flags
2198  * @param netdev network device
2199  *
2200  * This routine generates a octnet_ifflags mask from the net device flags
2201  * received from the OS.
2202  */
2203 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2204 {
2205         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2206
2207         if (netdev->flags & IFF_PROMISC)
2208                 f |= OCTNET_IFFLAG_PROMISC;
2209
2210         if (netdev->flags & IFF_ALLMULTI)
2211                 f |= OCTNET_IFFLAG_ALLMULTI;
2212
2213         if (netdev->flags & IFF_MULTICAST) {
2214                 f |= OCTNET_IFFLAG_MULTICAST;
2215
2216                 /* Accept all multicast addresses if there are more than we
2217                  * can handle
2218                  */
2219                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2220                         f |= OCTNET_IFFLAG_ALLMULTI;
2221         }
2222
2223         if (netdev->flags & IFF_BROADCAST)
2224                 f |= OCTNET_IFFLAG_BROADCAST;
2225
2226         return f;
2227 }
2228
2229 /**
2230  * \brief Net device set_multicast_list
2231  * @param netdev network device
2232  */
2233 static void liquidio_set_mcast_list(struct net_device *netdev)
2234 {
2235         struct lio *lio = GET_LIO(netdev);
2236         struct octeon_device *oct = lio->oct_dev;
2237         struct octnic_ctrl_pkt nctrl;
2238         struct octnic_ctrl_params nparams;
2239         struct netdev_hw_addr *ha;
2240         u64 *mc;
2241         int ret, i;
2242         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2243
2244         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2245
2246         /* Create a ctrl pkt command to be sent to core app. */
2247         nctrl.ncmd.u64 = 0;
2248         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2249         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2250         nctrl.ncmd.s.param2 = get_new_flags(netdev);
2251         nctrl.ncmd.s.param3 = mc_count;
2252         nctrl.ncmd.s.more = mc_count;
2253         nctrl.netpndev = (u64)netdev;
2254         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2255
2256         /* copy all the addresses into the udd */
2257         i = 0;
2258         mc = &nctrl.udd[0];
2259         netdev_for_each_mc_addr(ha, netdev) {
2260                 *mc = 0;
2261                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2262                 /* no need to swap bytes */
2263
2264                 if (++mc > &nctrl.udd[mc_count])
2265                         break;
2266         }
2267
2268         /* Apparently, any activity in this call from the kernel has to
2269          * be atomic. So we won't wait for response.
2270          */
2271         nctrl.wait_time = 0;
2272
2273         nparams.resp_order = OCTEON_RESP_NORESPONSE;
2274
2275         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2276         if (ret < 0) {
2277                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2278                         ret);
2279         }
2280 }
2281
2282 /**
2283  * \brief Net device set_mac_address
2284  * @param netdev network device
2285  */
2286 static int liquidio_set_mac(struct net_device *netdev, void *p)
2287 {
2288         int ret = 0;
2289         struct lio *lio = GET_LIO(netdev);
2290         struct octeon_device *oct = lio->oct_dev;
2291         struct sockaddr *addr = (struct sockaddr *)p;
2292         struct octnic_ctrl_pkt nctrl;
2293         struct octnic_ctrl_params nparams;
2294
2295         if ((!is_valid_ether_addr(addr->sa_data)) ||
2296             (ifstate_check(lio, LIO_IFSTATE_RUNNING)))
2297                 return -EADDRNOTAVAIL;
2298
2299         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2300
2301         nctrl.ncmd.u64 = 0;
2302         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2303         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2304         nctrl.ncmd.s.param2 = 0;
2305         nctrl.ncmd.s.more = 1;
2306         nctrl.netpndev = (u64)netdev;
2307         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2308         nctrl.wait_time = 100;
2309
2310         nctrl.udd[0] = 0;
2311         /* The MAC Address is presented in network byte order. */
2312         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2313
2314         nparams.resp_order = OCTEON_RESP_ORDERED;
2315
2316         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2317         if (ret < 0) {
2318                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2319                 return -ENOMEM;
2320         }
2321         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2322         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2323
2324         return 0;
2325 }
2326
2327 /**
2328  * \brief Net device get_stats
2329  * @param netdev network device
2330  */
2331 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2332 {
2333         struct lio *lio = GET_LIO(netdev);
2334         struct net_device_stats *stats = &netdev->stats;
2335         struct octeon_device *oct;
2336         u64 pkts = 0, drop = 0, bytes = 0;
2337         struct oct_droq_stats *oq_stats;
2338         struct oct_iq_stats *iq_stats;
2339         int i, iq_no, oq_no;
2340
2341         oct = lio->oct_dev;
2342
2343         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2344                 iq_no = lio->linfo.txpciq[i];
2345                 iq_stats = &oct->instr_queue[iq_no]->stats;
2346                 pkts += iq_stats->tx_done;
2347                 drop += iq_stats->tx_dropped;
2348                 bytes += iq_stats->tx_tot_bytes;
2349         }
2350
2351         stats->tx_packets = pkts;
2352         stats->tx_bytes = bytes;
2353         stats->tx_dropped = drop;
2354
2355         pkts = 0;
2356         drop = 0;
2357         bytes = 0;
2358
2359         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2360                 oq_no = lio->linfo.rxpciq[i];
2361                 oq_stats = &oct->droq[oq_no]->stats;
2362                 pkts += oq_stats->rx_pkts_received;
2363                 drop += (oq_stats->rx_dropped +
2364                          oq_stats->dropped_nodispatch +
2365                          oq_stats->dropped_toomany +
2366                          oq_stats->dropped_nomem);
2367                 bytes += oq_stats->rx_bytes_received;
2368         }
2369
2370         stats->rx_bytes = bytes;
2371         stats->rx_packets = pkts;
2372         stats->rx_dropped = drop;
2373
2374         return stats;
2375 }
2376
2377 /**
2378  * \brief Net device change_mtu
2379  * @param netdev network device
2380  */
2381 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2382 {
2383         struct lio *lio = GET_LIO(netdev);
2384         struct octeon_device *oct = lio->oct_dev;
2385         struct octnic_ctrl_pkt nctrl;
2386         struct octnic_ctrl_params nparams;
2387         int max_frm_size = new_mtu + OCTNET_FRM_HEADER_SIZE;
2388         int ret = 0;
2389
2390         /* Limit the MTU to make sure the ethernet packets are between 64 bytes
2391          * and 65535 bytes
2392          */
2393         if ((max_frm_size < OCTNET_MIN_FRM_SIZE) ||
2394             (max_frm_size > OCTNET_MAX_FRM_SIZE)) {
2395                 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2396                 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2397                         (OCTNET_MIN_FRM_SIZE - OCTNET_FRM_HEADER_SIZE),
2398                         (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE));
2399                 return -EINVAL;
2400         }
2401
2402         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2403
2404         nctrl.ncmd.u64 = 0;
2405         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2406         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2407         nctrl.ncmd.s.param2 = new_mtu;
2408         nctrl.wait_time = 100;
2409         nctrl.netpndev = (u64)netdev;
2410         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2411
2412         nparams.resp_order = OCTEON_RESP_ORDERED;
2413
2414         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2415         if (ret < 0) {
2416                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2417                 return -1;
2418         }
2419
2420         lio->mtu = new_mtu;
2421
2422         return 0;
2423 }
2424
2425 /**
2426  * \brief Handler for SIOCSHWTSTAMP ioctl
2427  * @param netdev network device
2428  * @param ifr interface request
2429  * @param cmd command
2430  */
2431 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2432 {
2433         struct hwtstamp_config conf;
2434         struct lio *lio = GET_LIO(netdev);
2435
2436         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2437                 return -EFAULT;
2438
2439         if (conf.flags)
2440                 return -EINVAL;
2441
2442         switch (conf.tx_type) {
2443         case HWTSTAMP_TX_ON:
2444         case HWTSTAMP_TX_OFF:
2445                 break;
2446         default:
2447                 return -ERANGE;
2448         }
2449
2450         switch (conf.rx_filter) {
2451         case HWTSTAMP_FILTER_NONE:
2452                 break;
2453         case HWTSTAMP_FILTER_ALL:
2454         case HWTSTAMP_FILTER_SOME:
2455         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2456         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2457         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2458         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2459         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2460         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2461         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2462         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2463         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2464         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2465         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2466         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2467                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2468                 break;
2469         default:
2470                 return -ERANGE;
2471         }
2472
2473         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2474                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2475
2476         else
2477                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2478
2479         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2480 }
2481
2482 /**
2483  * \brief ioctl handler
2484  * @param netdev network device
2485  * @param ifr interface request
2486  * @param cmd command
2487  */
2488 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2489 {
2490         switch (cmd) {
2491         case SIOCSHWTSTAMP:
2492                 return hwtstamp_ioctl(netdev, ifr, cmd);
2493         default:
2494                 return -EOPNOTSUPP;
2495         }
2496 }
2497
2498 /**
2499  * \brief handle a Tx timestamp response
2500  * @param status response status
2501  * @param buf pointer to skb
2502  */
2503 static void handle_timestamp(struct octeon_device *oct,
2504                              u32 status,
2505                              void *buf)
2506 {
2507         struct octnet_buf_free_info *finfo;
2508         struct octeon_soft_command *sc;
2509         struct oct_timestamp_resp *resp;
2510         struct lio *lio;
2511         struct sk_buff *skb = (struct sk_buff *)buf;
2512
2513         finfo = (struct octnet_buf_free_info *)skb->cb;
2514         lio = finfo->lio;
2515         sc = finfo->sc;
2516         oct = lio->oct_dev;
2517         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2518
2519         if (status != OCTEON_REQUEST_DONE) {
2520                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2521                         CVM_CAST64(status));
2522                 resp->timestamp = 0;
2523         }
2524
2525         octeon_swap_8B_data(&resp->timestamp, 1);
2526
2527         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2528                 struct skb_shared_hwtstamps ts;
2529                 u64 ns = resp->timestamp;
2530
2531                 netif_info(lio, tx_done, lio->netdev,
2532                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2533                            skb, (unsigned long long)ns);
2534                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2535                 skb_tstamp_tx(skb, &ts);
2536         }
2537
2538         octeon_free_soft_command(oct, sc);
2539         recv_buffer_free(skb);
2540 }
2541
2542 /* \brief Send a data packet that will be timestamped
2543  * @param oct octeon device
2544  * @param ndata pointer to network data
2545  * @param finfo pointer to private network data
2546  */
2547 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2548                                          struct octnic_data_pkt *ndata,
2549                                          struct octnet_buf_free_info *finfo,
2550                                          int xmit_more)
2551 {
2552         int retval;
2553         struct octeon_soft_command *sc;
2554         struct octeon_instr_ih *ih;
2555         struct octeon_instr_rdp *rdp;
2556         struct lio *lio;
2557         int ring_doorbell;
2558
2559         lio = finfo->lio;
2560
2561         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2562                                             sizeof(struct oct_timestamp_resp));
2563         finfo->sc = sc;
2564
2565         if (!sc) {
2566                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2567                 return IQ_SEND_FAILED;
2568         }
2569
2570         if (ndata->reqtype == REQTYPE_NORESP_NET)
2571                 ndata->reqtype = REQTYPE_RESP_NET;
2572         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2573                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2574
2575         sc->callback = handle_timestamp;
2576         sc->callback_arg = finfo->skb;
2577         sc->iq_no = ndata->q_no;
2578
2579         ih = (struct octeon_instr_ih *)&sc->cmd.ih;
2580         rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp;
2581
2582         ring_doorbell = !xmit_more;
2583         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2584                                      sc, ih->dlengsz, ndata->reqtype);
2585
2586         if (retval) {
2587                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2588                         retval);
2589                 octeon_free_soft_command(oct, sc);
2590         } else {
2591                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2592         }
2593
2594         return retval;
2595 }
2596
2597 static inline int is_ipv4(struct sk_buff *skb)
2598 {
2599         return (skb->protocol == htons(ETH_P_IP)) &&
2600                (ip_hdr(skb)->version == 4);
2601 }
2602
2603 static inline int is_vlan(struct sk_buff *skb)
2604 {
2605         return skb->protocol == htons(ETH_P_8021Q);
2606 }
2607
2608 static inline int is_ip_fragmented(struct sk_buff *skb)
2609 {
2610         /* The Don't fragment and Reserved flag fields are ignored.
2611          * IP is fragmented if
2612          * -  the More fragments bit is set (indicating this IP is a fragment
2613          * with more to follow; the current offset could be 0 ).
2614          * -  ths offset field is non-zero.
2615          */
2616         return (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ? 1 : 0;
2617 }
2618
2619 static inline int is_ipv6(struct sk_buff *skb)
2620 {
2621         return (skb->protocol == htons(ETH_P_IPV6)) &&
2622                (ipv6_hdr(skb)->version == 6);
2623 }
2624
2625 static inline int is_with_extn_hdr(struct sk_buff *skb)
2626 {
2627         return (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) &&
2628                (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP);
2629 }
2630
2631 static inline int is_tcpudp(struct sk_buff *skb)
2632 {
2633         return (ip_hdr(skb)->protocol == IPPROTO_TCP) ||
2634                (ip_hdr(skb)->protocol == IPPROTO_UDP);
2635 }
2636
2637 static inline u32 get_ipv4_5tuple_tag(struct sk_buff *skb)
2638 {
2639         u32 tag;
2640         struct iphdr *iphdr = ip_hdr(skb);
2641
2642         tag = crc32(0, &iphdr->protocol, 1);
2643         tag = crc32(tag, (u8 *)&iphdr->saddr, 8);
2644         tag = crc32(tag, skb_transport_header(skb), 4);
2645         return tag;
2646 }
2647
2648 static inline u32 get_ipv6_5tuple_tag(struct sk_buff *skb)
2649 {
2650         u32 tag;
2651         struct ipv6hdr *ipv6hdr = ipv6_hdr(skb);
2652
2653         tag = crc32(0, &ipv6hdr->nexthdr, 1);
2654         tag = crc32(tag, (u8 *)&ipv6hdr->saddr, 32);
2655         tag = crc32(tag, skb_transport_header(skb), 4);
2656         return tag;
2657 }
2658
2659 /** \brief Transmit networks packets to the Octeon interface
2660  * @param skbuff   skbuff struct to be passed to network layer.
2661  * @param netdev    pointer to network device
2662  * @returns whether the packet was transmitted to the device okay or not
2663  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2664  */
2665 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2666 {
2667         struct lio *lio;
2668         struct octnet_buf_free_info *finfo;
2669         union octnic_cmd_setup cmdsetup;
2670         struct octnic_data_pkt ndata;
2671         struct octeon_device *oct;
2672         struct oct_iq_stats *stats;
2673         int cpu = 0, status = 0;
2674         int q_idx = 0, iq_no = 0;
2675         int xmit_more;
2676         u32 tag = 0;
2677
2678         lio = GET_LIO(netdev);
2679         oct = lio->oct_dev;
2680
2681         if (netif_is_multiqueue(netdev)) {
2682                 cpu = skb->queue_mapping;
2683                 q_idx = (cpu & (lio->linfo.num_txpciq - 1));
2684                 iq_no = lio->linfo.txpciq[q_idx];
2685         } else {
2686                 iq_no = lio->txq;
2687         }
2688
2689         stats = &oct->instr_queue[iq_no]->stats;
2690
2691         /* Check for all conditions in which the current packet cannot be
2692          * transmitted.
2693          */
2694         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2695             (!lio->linfo.link.s.status) ||
2696             (skb->len <= 0)) {
2697                 netif_info(lio, tx_err, lio->netdev,
2698                            "Transmit failed link_status : %d\n",
2699                            lio->linfo.link.s.status);
2700                 goto lio_xmit_failed;
2701         }
2702
2703         /* Use space in skb->cb to store info used to unmap and
2704          * free the buffers.
2705          */
2706         finfo = (struct octnet_buf_free_info *)skb->cb;
2707         finfo->lio = lio;
2708         finfo->skb = skb;
2709         finfo->sc = NULL;
2710
2711         /* Prepare the attributes for the data to be passed to OSI. */
2712         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2713
2714         ndata.buf = (void *)finfo;
2715
2716         ndata.q_no = iq_no;
2717
2718         if (netif_is_multiqueue(netdev)) {
2719                 if (octnet_iq_is_full(oct, ndata.q_no)) {
2720                         /* defer sending if queue is full */
2721                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2722                                    ndata.q_no);
2723                         stats->tx_iq_busy++;
2724                         return NETDEV_TX_BUSY;
2725                 }
2726         } else {
2727                 if (octnet_iq_is_full(oct, lio->txq)) {
2728                         /* defer sending if queue is full */
2729                         stats->tx_iq_busy++;
2730                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2731                                    ndata.q_no);
2732                         return NETDEV_TX_BUSY;
2733                 }
2734         }
2735         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2736          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no );
2737          */
2738
2739         ndata.datasize = skb->len;
2740
2741         cmdsetup.u64 = 0;
2742         cmdsetup.s.ifidx = lio->linfo.ifidx;
2743
2744         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2745                 if (is_ipv4(skb) && !is_ip_fragmented(skb) && is_tcpudp(skb)) {
2746                         tag = get_ipv4_5tuple_tag(skb);
2747
2748                         cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
2749
2750                         if (ip_hdr(skb)->ihl > 5)
2751                                 cmdsetup.s.ipv4opts_ipv6exthdr =
2752                                                 OCT_PKT_PARAM_IPV4OPTS;
2753
2754                 } else if (is_ipv6(skb)) {
2755                         tag = get_ipv6_5tuple_tag(skb);
2756
2757                         cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
2758
2759                         if (is_with_extn_hdr(skb))
2760                                 cmdsetup.s.ipv4opts_ipv6exthdr =
2761                                                 OCT_PKT_PARAM_IPV6EXTHDR;
2762
2763                 } else if (is_vlan(skb)) {
2764                         if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
2765                                 == htons(ETH_P_IP) &&
2766                                 !is_ip_fragmented(skb) && is_tcpudp(skb)) {
2767                                 tag = get_ipv4_5tuple_tag(skb);
2768
2769                                 cmdsetup.s.cksum_offset =
2770                                         sizeof(struct vlan_ethhdr) + 1;
2771
2772                                 if (ip_hdr(skb)->ihl > 5)
2773                                         cmdsetup.s.ipv4opts_ipv6exthdr =
2774                                                 OCT_PKT_PARAM_IPV4OPTS;
2775
2776                         } else if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
2777                                 == htons(ETH_P_IPV6)) {
2778                                 tag = get_ipv6_5tuple_tag(skb);
2779
2780                                 cmdsetup.s.cksum_offset =
2781                                         sizeof(struct vlan_ethhdr) + 1;
2782
2783                                 if (is_with_extn_hdr(skb))
2784                                         cmdsetup.s.ipv4opts_ipv6exthdr =
2785                                                 OCT_PKT_PARAM_IPV6EXTHDR;
2786                         }
2787                 }
2788         }
2789         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2790                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2791                 cmdsetup.s.timestamp = 1;
2792         }
2793
2794         if (skb_shinfo(skb)->nr_frags == 0) {
2795                 cmdsetup.s.u.datasize = skb->len;
2796                 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
2797                 /* Offload checksum calculation for TCP/UDP packets */
2798                 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
2799                                                 skb->data,
2800                                                 skb->len,
2801                                                 DMA_TO_DEVICE);
2802                 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
2803                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2804                                 __func__);
2805                         return NETDEV_TX_BUSY;
2806                 }
2807
2808                 finfo->dptr = ndata.cmd.dptr;
2809
2810                 ndata.reqtype = REQTYPE_NORESP_NET;
2811
2812         } else {
2813                 int i, frags;
2814                 struct skb_frag_struct *frag;
2815                 struct octnic_gather *g;
2816
2817                 spin_lock(&lio->lock);
2818                 g = (struct octnic_gather *)list_delete_head(&lio->glist);
2819                 spin_unlock(&lio->lock);
2820
2821                 if (!g) {
2822                         netif_info(lio, tx_err, lio->netdev,
2823                                    "Transmit scatter gather: glist null!\n");
2824                         goto lio_xmit_failed;
2825                 }
2826
2827                 cmdsetup.s.gather = 1;
2828                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2829                 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
2830
2831                 memset(g->sg, 0, g->sg_size);
2832
2833                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2834                                                  skb->data,
2835                                                  (skb->len - skb->data_len),
2836                                                  DMA_TO_DEVICE);
2837                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2838                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2839                                 __func__);
2840                         return NETDEV_TX_BUSY;
2841                 }
2842                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2843
2844                 frags = skb_shinfo(skb)->nr_frags;
2845                 i = 1;
2846                 while (frags--) {
2847                         frag = &skb_shinfo(skb)->frags[i - 1];
2848
2849                         g->sg[(i >> 2)].ptr[(i & 3)] =
2850                                 dma_map_page(&oct->pci_dev->dev,
2851                                              frag->page.p,
2852                                              frag->page_offset,
2853                                              frag->size,
2854                                              DMA_TO_DEVICE);
2855
2856                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2857                         i++;
2858                 }
2859
2860                 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
2861                                                 g->sg, g->sg_size,
2862                                                 DMA_TO_DEVICE);
2863                 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
2864                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2865                                 __func__);
2866                         dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0],
2867                                          skb->len - skb->data_len,
2868                                          DMA_TO_DEVICE);
2869                         return NETDEV_TX_BUSY;
2870                 }
2871
2872                 finfo->dptr = ndata.cmd.dptr;
2873                 finfo->g = g;
2874
2875                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2876         }
2877
2878         if (skb_shinfo(skb)->gso_size) {
2879                 struct octeon_instr_irh *irh =
2880                         (struct octeon_instr_irh *)&ndata.cmd.irh;
2881                 union tx_info *tx_info = (union tx_info *)&ndata.cmd.ossp[0];
2882
2883                 irh->len = 1;   /* to indicate that ossp[0] contains tx_info */
2884                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2885                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2886         }
2887
2888         xmit_more = skb->xmit_more;
2889
2890         if (unlikely(cmdsetup.s.timestamp))
2891                 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2892         else
2893                 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2894         if (status == IQ_SEND_FAILED)
2895                 goto lio_xmit_dma_failed;
2896
2897         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2898
2899         if (status == IQ_SEND_STOP)
2900                 stop_q(lio->netdev, q_idx);
2901
2902         netdev->trans_start = jiffies;
2903
2904         stats->tx_done++;
2905         stats->tx_tot_bytes += skb->len;
2906
2907         return NETDEV_TX_OK;
2908
2909 lio_xmit_dma_failed:
2910         dma_unmap_single(&oct->pci_dev->dev, ndata.cmd.dptr,
2911                          ndata.datasize, DMA_TO_DEVICE);
2912 lio_xmit_failed:
2913         stats->tx_dropped++;
2914         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2915                    iq_no, stats->tx_dropped);
2916         recv_buffer_free(skb);
2917         return NETDEV_TX_OK;
2918 }
2919
2920 /** \brief Network device Tx timeout
2921  * @param netdev    pointer to network device
2922  */
2923 static void liquidio_tx_timeout(struct net_device *netdev)
2924 {
2925         struct lio *lio;
2926
2927         lio = GET_LIO(netdev);
2928
2929         netif_info(lio, tx_err, lio->netdev,
2930                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2931                    netdev->stats.tx_dropped);
2932         netdev->trans_start = jiffies;
2933         txqs_wake(netdev);
2934 }
2935
2936 int liquidio_set_feature(struct net_device *netdev, int cmd)
2937 {
2938         struct lio *lio = GET_LIO(netdev);
2939         struct octeon_device *oct = lio->oct_dev;
2940         struct octnic_ctrl_pkt nctrl;
2941         struct octnic_ctrl_params nparams;
2942         int ret = 0;
2943
2944         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2945
2946         nctrl.ncmd.u64 = 0;
2947         nctrl.ncmd.s.cmd = cmd;
2948         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2949         nctrl.ncmd.s.param2 = OCTNIC_LROIPV4 | OCTNIC_LROIPV6;
2950         nctrl.wait_time = 100;
2951         nctrl.netpndev = (u64)netdev;
2952         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2953
2954         nparams.resp_order = OCTEON_RESP_NORESPONSE;
2955
2956         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2957         if (ret < 0) {
2958                 dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
2959                         ret);
2960         }
2961         return ret;
2962 }
2963
2964 /** \brief Net device fix features
2965  * @param netdev  pointer to network device
2966  * @param request features requested
2967  * @returns updated features list
2968  */
2969 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2970                                                netdev_features_t request)
2971 {
2972         struct lio *lio = netdev_priv(netdev);
2973
2974         if ((request & NETIF_F_RXCSUM) &&
2975             !(lio->dev_capability & NETIF_F_RXCSUM))
2976                 request &= ~NETIF_F_RXCSUM;
2977
2978         if ((request & NETIF_F_HW_CSUM) &&
2979             !(lio->dev_capability & NETIF_F_HW_CSUM))
2980                 request &= ~NETIF_F_HW_CSUM;
2981
2982         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2983                 request &= ~NETIF_F_TSO;
2984
2985         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2986                 request &= ~NETIF_F_TSO6;
2987
2988         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2989                 request &= ~NETIF_F_LRO;
2990
2991         /*Disable LRO if RXCSUM is off */
2992         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2993             (lio->dev_capability & NETIF_F_LRO))
2994                 request &= ~NETIF_F_LRO;
2995
2996         return request;
2997 }
2998
2999 /** \brief Net device set features
3000  * @param netdev  pointer to network device
3001  * @param features features to enable/disable
3002  */
3003 static int liquidio_set_features(struct net_device *netdev,
3004                                  netdev_features_t features)
3005 {
3006         struct lio *lio = netdev_priv(netdev);
3007
3008         if (!((netdev->features ^ features) & NETIF_F_LRO))
3009                 return 0;
3010
3011         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3012                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
3013         else if (!(features & NETIF_F_LRO) &&
3014                  (lio->dev_capability & NETIF_F_LRO))
3015                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE);
3016
3017         return 0;
3018 }
3019
3020 static struct net_device_ops lionetdevops = {
3021         .ndo_open               = liquidio_open,
3022         .ndo_stop               = liquidio_stop,
3023         .ndo_start_xmit         = liquidio_xmit,
3024         .ndo_get_stats          = liquidio_get_stats,
3025         .ndo_set_mac_address    = liquidio_set_mac,
3026         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3027         .ndo_tx_timeout         = liquidio_tx_timeout,
3028         .ndo_change_mtu         = liquidio_change_mtu,
3029         .ndo_do_ioctl           = liquidio_ioctl,
3030         .ndo_fix_features       = liquidio_fix_features,
3031         .ndo_set_features       = liquidio_set_features,
3032 };
3033
3034 /** \brief Entry point for the liquidio module
3035  */
3036 static int __init liquidio_init(void)
3037 {
3038         int i;
3039         struct handshake *hs;
3040
3041         init_completion(&first_stage);
3042
3043         octeon_init_device_list(conf_type);
3044
3045         if (liquidio_init_pci())
3046                 return -EINVAL;
3047
3048         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3049
3050         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3051                 hs = &handshake[i];
3052                 if (hs->pci_dev) {
3053                         wait_for_completion(&hs->init);
3054                         if (!hs->init_ok) {
3055                                 /* init handshake failed */
3056                                 dev_err(&hs->pci_dev->dev,
3057                                         "Failed to init device\n");
3058                                 liquidio_deinit_pci();
3059                                 return -EIO;
3060                         }
3061                 }
3062         }
3063
3064         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3065                 hs = &handshake[i];
3066                 if (hs->pci_dev) {
3067                         wait_for_completion_timeout(&hs->started,
3068                                                     msecs_to_jiffies(30000));
3069                         if (!hs->started_ok) {
3070                                 /* starter handshake failed */
3071                                 dev_err(&hs->pci_dev->dev,
3072                                         "Firmware failed to start\n");
3073                                 liquidio_deinit_pci();
3074                                 return -EIO;
3075                         }
3076                 }
3077         }
3078
3079         return 0;
3080 }
3081
3082 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3083 {
3084         struct octeon_device *oct = (struct octeon_device *)buf;
3085         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3086         int ifidx = 0;
3087         union oct_link_status *ls;
3088         int i;
3089
3090         if ((recv_pkt->buffer_size[0] != sizeof(*ls)) ||
3091             (recv_pkt->rh.r_nic_info.ifidx > oct->ifcount)) {
3092                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3093                         recv_pkt->buffer_size[0],
3094                         recv_pkt->rh.r_nic_info.ifidx);
3095                 goto nic_info_err;
3096         }
3097
3098         ifidx = recv_pkt->rh.r_nic_info.ifidx;
3099         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3100
3101         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3102
3103         update_link_status(oct->props[ifidx].netdev, ls);
3104
3105 nic_info_err:
3106         for (i = 0; i < recv_pkt->buffer_count; i++)
3107                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3108         octeon_free_recv_info(recv_info);
3109         return 0;
3110 }
3111
3112 /**
3113  * \brief Setup network interfaces
3114  * @param octeon_dev  octeon device
3115  *
3116  * Called during init time for each device. It assumes the NIC
3117  * is already up and running.  The link information for each
3118  * interface is passed in link_info.
3119  */
3120 static int setup_nic_devices(struct octeon_device *octeon_dev)
3121 {
3122         struct lio *lio = NULL;
3123         struct net_device *netdev;
3124         u8 mac[6], i, j;
3125         struct octeon_soft_command *sc;
3126         struct liquidio_if_cfg_context *ctx;
3127         struct liquidio_if_cfg_resp *resp;
3128         struct octdev_props *props;
3129         int retval, num_iqueues, num_oqueues, q_no;
3130         u64 q_mask;
3131         int num_cpus = num_online_cpus();
3132         union oct_nic_if_cfg if_cfg;
3133         unsigned int base_queue;
3134         unsigned int gmx_port_id;
3135         u32 resp_size, ctx_size;
3136
3137         /* This is to handle link status changes */
3138         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3139                                     OPCODE_NIC_INFO,
3140                                     lio_nic_info, octeon_dev);
3141
3142         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3143          * They are handled directly.
3144          */
3145         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3146                                         free_netbuf);
3147
3148         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3149                                         free_netsgbuf);
3150
3151         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3152                                         free_netsgbuf_with_resp);
3153
3154         for (i = 0; i < octeon_dev->ifcount; i++) {
3155                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3156                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3157                 sc = (struct octeon_soft_command *)
3158                         octeon_alloc_soft_command(octeon_dev, 0,
3159                                                   resp_size, ctx_size);
3160                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3161                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3162
3163                 num_iqueues =
3164                         CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3165                 num_oqueues =
3166                         CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3167                 base_queue =
3168                         CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i);
3169                 gmx_port_id =
3170                         CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i);
3171                 if (num_iqueues > num_cpus)
3172                         num_iqueues = num_cpus;
3173                 if (num_oqueues > num_cpus)
3174                         num_oqueues = num_cpus;
3175                 dev_dbg(&octeon_dev->pci_dev->dev,
3176                         "requesting config for interface %d, iqs %d, oqs %d\n",
3177                         i, num_iqueues, num_oqueues);
3178                 ACCESS_ONCE(ctx->cond) = 0;
3179                 ctx->octeon_id = lio_get_device_id(octeon_dev);
3180                 init_waitqueue_head(&ctx->wc);
3181
3182                 if_cfg.u64 = 0;
3183                 if_cfg.s.num_iqueues = num_iqueues;
3184                 if_cfg.s.num_oqueues = num_oqueues;
3185                 if_cfg.s.base_queue = base_queue;
3186                 if_cfg.s.gmx_port_id = gmx_port_id;
3187                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3188                                             OPCODE_NIC_IF_CFG, i,
3189                                             if_cfg.u64, 0);
3190
3191                 sc->callback = if_cfg_callback;
3192                 sc->callback_arg = sc;
3193                 sc->wait_time = 1000;
3194
3195                 retval = octeon_send_soft_command(octeon_dev, sc);
3196                 if (retval) {
3197                         dev_err(&octeon_dev->pci_dev->dev,
3198                                 "iq/oq config failed status: %x\n",
3199                                 retval);
3200                         /* Soft instr is freed by driver in case of failure. */
3201                         goto setup_nic_dev_fail;
3202                 }
3203
3204                 /* Sleep on a wait queue till the cond flag indicates that the
3205                  * response arrived or timed-out.
3206                  */
3207                 sleep_cond(&ctx->wc, &ctx->cond);
3208                 retval = resp->status;
3209                 if (retval) {
3210                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3211                         goto setup_nic_dev_fail;
3212                 }
3213
3214                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3215                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3216
3217                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3218                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3219
3220                 if (!(num_iqueues) || !(num_oqueues)) {
3221                         dev_err(&octeon_dev->pci_dev->dev,
3222                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3223                                 resp->cfg_info.iqmask,
3224                                 resp->cfg_info.oqmask);
3225                         goto setup_nic_dev_fail;
3226                 }
3227                 dev_dbg(&octeon_dev->pci_dev->dev,
3228                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3229                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3230                         num_iqueues, num_oqueues);
3231                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3232
3233                 if (!netdev) {
3234                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3235                         goto setup_nic_dev_fail;
3236                 }
3237
3238                 props = &octeon_dev->props[i];
3239                 props->netdev = netdev;
3240
3241                 if (num_iqueues > 1)
3242                         lionetdevops.ndo_select_queue = select_q;
3243
3244                 /* Associate the routines that will handle different
3245                  * netdev tasks.
3246                  */
3247                 netdev->netdev_ops = &lionetdevops;
3248
3249                 lio = GET_LIO(netdev);
3250
3251                 memset(lio, 0, sizeof(struct lio));
3252
3253                 lio->linfo.ifidx = resp->cfg_info.ifidx;
3254                 lio->ifidx = resp->cfg_info.ifidx;
3255
3256                 lio->linfo.num_rxpciq = num_oqueues;
3257                 lio->linfo.num_txpciq = num_iqueues;
3258                 q_mask = resp->cfg_info.oqmask;
3259                 /* q_mask is 0-based and already verified mask is nonzero */
3260                 for (j = 0; j < num_oqueues; j++) {
3261                         q_no = __ffs64(q_mask);
3262                         q_mask &= (~(1UL << q_no));
3263                         lio->linfo.rxpciq[j] = q_no;
3264                 }
3265                 q_mask = resp->cfg_info.iqmask;
3266                 for (j = 0; j < num_iqueues; j++) {
3267                         q_no = __ffs64(q_mask);
3268                         q_mask &= (~(1UL << q_no));
3269                         lio->linfo.txpciq[j] = q_no;
3270                 }
3271                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3272                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3273                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3274
3275                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3276
3277                 lio->dev_capability = NETIF_F_HIGHDMA
3278                                       | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
3279                                       | NETIF_F_SG | NETIF_F_RXCSUM
3280                                       | NETIF_F_TSO | NETIF_F_TSO6
3281                                       | NETIF_F_LRO;
3282                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3283
3284                 netdev->features = lio->dev_capability;
3285                 netdev->vlan_features = lio->dev_capability;
3286
3287                 netdev->hw_features = lio->dev_capability;
3288
3289                 /* Point to the  properties for octeon device to which this
3290                  * interface belongs.
3291                  */
3292                 lio->oct_dev = octeon_dev;
3293                 lio->octprops = props;
3294                 lio->netdev = netdev;
3295                 spin_lock_init(&lio->lock);
3296
3297                 dev_dbg(&octeon_dev->pci_dev->dev,
3298                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3299                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3300
3301                 /* 64-bit swap required on LE machines */
3302                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3303                 for (j = 0; j < 6; j++)
3304                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3305
3306                 /* Copy MAC Address to OS network device structure */
3307
3308                 ether_addr_copy(netdev->dev_addr, mac);
3309
3310                 if (setup_io_queues(octeon_dev, netdev)) {
3311                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3312                         goto setup_nic_dev_fail;
3313                 }
3314
3315                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3316
3317                 /* By default all interfaces on a single Octeon uses the same
3318                  * tx and rx queues
3319                  */
3320                 lio->txq = lio->linfo.txpciq[0];
3321                 lio->rxq = lio->linfo.rxpciq[0];
3322
3323                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3324                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3325
3326                 if (setup_glist(lio)) {
3327                         dev_err(&octeon_dev->pci_dev->dev,
3328                                 "Gather list allocation failed\n");
3329                         goto setup_nic_dev_fail;
3330                 }
3331
3332                 /* Register ethtool support */
3333                 liquidio_set_ethtool_ops(netdev);
3334
3335                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
3336
3337                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3338                         liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE);
3339
3340                 /* Register the network device with the OS */
3341                 if (register_netdev(netdev)) {
3342                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3343                         goto setup_nic_dev_fail;
3344                 }
3345
3346                 dev_dbg(&octeon_dev->pci_dev->dev,
3347                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3348                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3349                 netif_carrier_off(netdev);
3350
3351                 if (lio->linfo.link.s.status) {
3352                         netif_carrier_on(netdev);
3353                         start_txq(netdev);
3354                 } else {
3355                         netif_carrier_off(netdev);
3356                 }
3357
3358                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3359
3360                 dev_dbg(&octeon_dev->pci_dev->dev,
3361                         "NIC ifidx:%d Setup successful\n", i);
3362
3363                 octeon_free_soft_command(octeon_dev, sc);
3364         }
3365
3366         return 0;
3367
3368 setup_nic_dev_fail:
3369
3370         octeon_free_soft_command(octeon_dev, sc);
3371
3372         while (i--) {
3373                 dev_err(&octeon_dev->pci_dev->dev,
3374                         "NIC ifidx:%d Setup failed\n", i);
3375                 liquidio_destroy_nic_device(octeon_dev, i);
3376         }
3377         return -ENODEV;
3378 }
3379
3380 /**
3381  * \brief initialize the NIC
3382  * @param oct octeon device
3383  *
3384  * This initialization routine is called once the Octeon device application is
3385  * up and running
3386  */
3387 static int liquidio_init_nic_module(struct octeon_device *oct)
3388 {
3389         struct oct_intrmod_cfg *intrmod_cfg;
3390         int retval = 0;
3391         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3392
3393         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3394
3395         /* only default iq and oq were initialized
3396          * initialize the rest as well
3397          */
3398         /* run port_config command for each port */
3399         oct->ifcount = num_nic_ports;
3400
3401         memset(oct->props, 0,
3402                sizeof(struct octdev_props) * num_nic_ports);
3403
3404         retval = setup_nic_devices(oct);
3405         if (retval) {
3406                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3407                 goto octnet_init_failure;
3408         }
3409
3410         liquidio_ptp_init(oct);
3411
3412         /* Initialize interrupt moderation params */
3413         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
3414         intrmod_cfg->intrmod_enable = 1;
3415         intrmod_cfg->intrmod_check_intrvl = LIO_INTRMOD_CHECK_INTERVAL;
3416         intrmod_cfg->intrmod_maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
3417         intrmod_cfg->intrmod_minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
3418         intrmod_cfg->intrmod_maxcnt_trigger = LIO_INTRMOD_MAXCNT_TRIGGER;
3419         intrmod_cfg->intrmod_maxtmr_trigger = LIO_INTRMOD_MAXTMR_TRIGGER;
3420         intrmod_cfg->intrmod_mintmr_trigger = LIO_INTRMOD_MINTMR_TRIGGER;
3421         intrmod_cfg->intrmod_mincnt_trigger = LIO_INTRMOD_MINCNT_TRIGGER;
3422
3423         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3424
3425         return retval;
3426
3427 octnet_init_failure:
3428
3429         oct->ifcount = 0;
3430
3431         return retval;
3432 }
3433
3434 /**
3435  * \brief starter callback that invokes the remaining initialization work after
3436  * the NIC is up and running.
3437  * @param octptr  work struct work_struct
3438  */
3439 static void nic_starter(struct work_struct *work)
3440 {
3441         struct octeon_device *oct;
3442         struct cavium_wk *wk = (struct cavium_wk *)work;
3443
3444         oct = (struct octeon_device *)wk->ctxptr;
3445
3446         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3447                 return;
3448
3449         /* If the status of the device is CORE_OK, the core
3450          * application has reported its application type. Call
3451          * any registered handlers now and move to the RUNNING
3452          * state.
3453          */
3454         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3455                 schedule_delayed_work(&oct->nic_poll_work.work,
3456                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3457                 return;
3458         }
3459
3460         atomic_set(&oct->status, OCT_DEV_RUNNING);
3461
3462         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3463                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3464
3465                 if (liquidio_init_nic_module(oct))
3466                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3467                 else
3468                         handshake[oct->octeon_id].started_ok = 1;
3469         } else {
3470                 dev_err(&oct->pci_dev->dev,
3471                         "Unexpected application running on NIC (%d). Check firmware.\n",
3472                         oct->app_mode);
3473         }
3474
3475         complete(&handshake[oct->octeon_id].started);
3476 }
3477
3478 /**
3479  * \brief Device initialization for each Octeon device that is probed
3480  * @param octeon_dev  octeon device
3481  */
3482 static int octeon_device_init(struct octeon_device *octeon_dev)
3483 {
3484         int j, ret;
3485         struct octeon_device_priv *oct_priv =
3486                 (struct octeon_device_priv *)octeon_dev->priv;
3487         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
3488
3489         /* Enable access to the octeon device and make its DMA capability
3490          * known to the OS.
3491          */
3492         if (octeon_pci_os_setup(octeon_dev))
3493                 return 1;
3494
3495         /* Identify the Octeon type and map the BAR address space. */
3496         if (octeon_chip_specific_setup(octeon_dev)) {
3497                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
3498                 return 1;
3499         }
3500
3501         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
3502
3503         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
3504
3505         /* Do a soft reset of the Octeon device. */
3506         if (octeon_dev->fn_list.soft_reset(octeon_dev))
3507                 return 1;
3508
3509         /* Initialize the dispatch mechanism used to push packets arriving on
3510          * Octeon Output queues.
3511          */
3512         if (octeon_init_dispatch_list(octeon_dev))
3513                 return 1;
3514
3515         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3516                                     OPCODE_NIC_CORE_DRV_ACTIVE,
3517                                     octeon_core_drv_init,
3518                                     octeon_dev);
3519
3520         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
3521         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
3522         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
3523                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3524
3525         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
3526
3527         octeon_set_io_queues_off(octeon_dev);
3528
3529         /*  Setup the data structures that manage this Octeon's Input queues. */
3530         if (octeon_setup_instr_queues(octeon_dev)) {
3531                 dev_err(&octeon_dev->pci_dev->dev,
3532                         "instruction queue initialization failed\n");
3533                 /* On error, release any previously allocated queues */
3534                 for (j = 0; j < octeon_dev->num_iqs; j++)
3535                         octeon_delete_instr_queue(octeon_dev, j);
3536                 return 1;
3537         }
3538         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
3539
3540         /* Initialize soft command buffer pool
3541          */
3542         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
3543                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
3544                 return 1;
3545         }
3546         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
3547
3548         /* Initialize lists to manage the requests of different types that
3549          * arrive from user & kernel applications for this octeon device.
3550          */
3551         if (octeon_setup_response_list(octeon_dev)) {
3552                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
3553                 return 1;
3554         }
3555         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
3556
3557         if (octeon_setup_output_queues(octeon_dev)) {
3558                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
3559                 /* Release any previously allocated queues */
3560                 for (j = 0; j < octeon_dev->num_oqs; j++)
3561                         octeon_delete_droq(octeon_dev, j);
3562         }
3563
3564         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
3565
3566         /* The input and output queue registers were setup earlier (the queues
3567          * were not enabled). Any additional registers that need to be
3568          * programmed should be done now.
3569          */
3570         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3571         if (ret) {
3572                 dev_err(&octeon_dev->pci_dev->dev,
3573                         "Failed to configure device registers\n");
3574                 return ret;
3575         }
3576
3577         /* Initialize the tasklet that handles output queue packet processing.*/
3578         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
3579         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
3580                      (unsigned long)octeon_dev);
3581
3582         /* Setup the interrupt handler and record the INT SUM register address
3583          */
3584         octeon_setup_interrupt(octeon_dev);
3585
3586         /* Enable Octeon device interrupts */
3587         octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
3588
3589         /* Enable the input and output queues for this Octeon device */
3590         octeon_dev->fn_list.enable_io_queues(octeon_dev);
3591
3592         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
3593
3594         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
3595
3596         if (ddr_timeout == 0) {
3597                 dev_info(&octeon_dev->pci_dev->dev,
3598                          "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3599         }
3600
3601         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
3602
3603         /* Wait for the octeon to initialize DDR after the soft-reset. */
3604         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
3605         if (ret) {
3606                 dev_err(&octeon_dev->pci_dev->dev,
3607                         "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3608                         ret);
3609                 return 1;
3610         }
3611
3612         if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
3613                 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
3614                 return 1;
3615         }
3616
3617         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
3618         ret = octeon_init_consoles(octeon_dev);
3619         if (ret) {
3620                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
3621                 return 1;
3622         }
3623         ret = octeon_add_console(octeon_dev, 0);
3624         if (ret) {
3625                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
3626                 return 1;
3627         }
3628
3629         atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
3630
3631         dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
3632         ret = load_firmware(octeon_dev);
3633         if (ret) {
3634                 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
3635                 return 1;
3636         }
3637
3638         handshake[octeon_dev->octeon_id].init_ok = 1;
3639         complete(&handshake[octeon_dev->octeon_id].init);
3640
3641         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
3642
3643         /* Send Credit for Octeon Output queues. Credits are always sent after
3644          * the output queue is enabled.
3645          */
3646         for (j = 0; j < octeon_dev->num_oqs; j++)
3647                 writel(octeon_dev->droq[j]->max_count,
3648                        octeon_dev->droq[j]->pkts_credit_reg);
3649
3650         /* Packets can start arriving on the output queues from this point. */
3651
3652         return 0;
3653 }
3654
3655 /**
3656  * \brief Exits the module
3657  */
3658 static void __exit liquidio_exit(void)
3659 {
3660         liquidio_deinit_pci();
3661
3662         pr_info("LiquidIO network module is now unloaded\n");
3663 }
3664
3665 module_init(liquidio_init);
3666 module_exit(liquidio_exit);