1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2016 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18 #include <linux/module.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <linux/firmware.h>
22 #include <net/vxlan.h>
23 #include <linux/kthread.h>
24 #include "liquidio_common.h"
25 #include "octeon_droq.h"
26 #include "octeon_iq.h"
27 #include "response_manager.h"
28 #include "octeon_device.h"
29 #include "octeon_nic.h"
30 #include "octeon_main.h"
31 #include "octeon_network.h"
32 #include "cn66xx_regs.h"
33 #include "cn66xx_device.h"
34 #include "cn68xx_device.h"
35 #include "cn23xx_pf_device.h"
36 #include "liquidio_image.h"
37 #include "lio_vf_rep.h"
39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
41 MODULE_LICENSE("GPL");
42 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
43 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
44 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
45 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
47 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
48 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
49 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
51 static int ddr_timeout = 10000;
52 module_param(ddr_timeout, int, 0644);
53 MODULE_PARM_DESC(ddr_timeout,
54 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
56 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
58 static int debug = -1;
59 module_param(debug, int, 0644);
60 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
62 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
63 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
64 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
66 static u32 console_bitmask;
67 module_param(console_bitmask, int, 0644);
68 MODULE_PARM_DESC(console_bitmask,
69 "Bitmask indicating which consoles have debug output redirected to syslog.");
72 * octeon_console_debug_enabled - determines if a given console has debug enabled.
73 * @console: console to check
74 * Return: 1 = enabled. 0 otherwise
76 static int octeon_console_debug_enabled(u32 console)
78 return (console_bitmask >> (console)) & 0x1;
81 /* Polling interval for determining when NIC application is alive */
82 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
84 /* runtime link query interval */
85 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000
86 /* update localtime to octeon firmware every 60 seconds.
87 * make firmware to use same time reference, so that it will be easy to
88 * correlate firmware logged events/errors with host events, for debugging.
90 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
92 /* time to wait for possible in-flight requests in milliseconds */
93 #define WAIT_INFLIGHT_REQUEST msecs_to_jiffies(1000)
95 struct oct_link_status_resp {
97 struct oct_link_info link_info;
101 struct oct_timestamp_resp {
107 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
112 #ifdef __BIG_ENDIAN_BITFIELD
124 /* Octeon device properties to be used by the NIC module.
125 * Each octeon device in the system will be represented
126 * by this structure in the NIC module.
129 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
130 #define OCTNIC_GSO_MAX_SIZE \
131 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
134 struct completion init;
135 struct completion started;
136 struct pci_dev *pci_dev;
141 #ifdef CONFIG_PCI_IOV
142 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
145 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
146 char *prefix, char *suffix);
148 static int octeon_device_init(struct octeon_device *);
149 static int liquidio_stop(struct net_device *netdev);
150 static void liquidio_remove(struct pci_dev *pdev);
151 static int liquidio_probe(struct pci_dev *pdev,
152 const struct pci_device_id *ent);
153 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
156 static struct handshake handshake[MAX_OCTEON_DEVICES];
157 static struct completion first_stage;
159 static void octeon_droq_bh(struct tasklet_struct *t)
163 struct octeon_device_priv *oct_priv = from_tasklet(oct_priv, t,
165 struct octeon_device *oct = oct_priv->dev;
167 for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
168 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
170 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
172 lio_enable_irq(oct->droq[q_no], NULL);
174 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
175 /* set time and cnt interrupt thresholds for this DROQ
178 int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
181 oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
184 oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
189 tasklet_schedule(&oct_priv->droq_tasklet);
192 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
194 struct octeon_device_priv *oct_priv = oct->priv;
195 int retry = 100, pkt_cnt = 0, pending_pkts = 0;
201 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
202 if (!(oct->io_qmask.oq & BIT_ULL(i)))
204 pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
207 pending_pkts += pkt_cnt;
208 tasklet_schedule(&oct_priv->droq_tasklet);
211 schedule_timeout_uninterruptible(1);
213 } while (retry-- && pending_pkts);
219 * force_io_queues_off - Forces all IO queues off on a given device
220 * @oct: Pointer to Octeon device
222 static void force_io_queues_off(struct octeon_device *oct)
224 if ((oct->chip_id == OCTEON_CN66XX) ||
225 (oct->chip_id == OCTEON_CN68XX)) {
226 /* Reset the Enable bits for Input Queues. */
227 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
229 /* Reset the Enable bits for Output Queues. */
230 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
235 * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc
236 * @oct: Pointer to Octeon device
238 static inline void pcierror_quiesce_device(struct octeon_device *oct)
242 /* Disable the input and output queues now. No more packets will
243 * arrive from Octeon, but we should wait for all packet processing
246 force_io_queues_off(oct);
248 /* To allow for in-flight requests */
249 schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
251 if (wait_for_pending_requests(oct))
252 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
254 /* Force all requests waiting to be fetched by OCTEON to complete. */
255 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
256 struct octeon_instr_queue *iq;
258 if (!(oct->io_qmask.iq & BIT_ULL(i)))
260 iq = oct->instr_queue[i];
262 if (atomic_read(&iq->instr_pending)) {
263 spin_lock_bh(&iq->lock);
265 iq->octeon_read_index = iq->host_write_index;
266 iq->stats.instr_processed +=
267 atomic_read(&iq->instr_pending);
268 lio_process_iq_request_list(oct, iq, 0);
269 spin_unlock_bh(&iq->lock);
273 /* Force all pending ordered list requests to time out. */
274 lio_process_ordered_list(oct, 1);
276 /* We do not need to wait for output queue packets to be processed. */
280 * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status
281 * @dev: Pointer to PCI device
283 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
288 pr_info("%s :\n", __func__);
290 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
291 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
292 if (dev->error_state == pci_channel_io_normal)
293 status &= ~mask; /* Clear corresponding nonfatal bits */
295 status &= mask; /* Clear corresponding fatal bits */
296 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
300 * stop_pci_io - Stop all PCI IO to a given device
301 * @oct: Pointer to Octeon device
303 static void stop_pci_io(struct octeon_device *oct)
305 /* No more instructions will be forwarded. */
306 atomic_set(&oct->status, OCT_DEV_IN_RESET);
308 pci_disable_device(oct->pci_dev);
310 /* Disable interrupts */
311 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
313 pcierror_quiesce_device(oct);
315 /* Release the interrupt line */
316 free_irq(oct->pci_dev->irq, oct);
318 if (oct->flags & LIO_FLAG_MSI_ENABLED)
319 pci_disable_msi(oct->pci_dev);
321 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
322 lio_get_state_string(&oct->status));
324 /* making it a common function for all OCTEON models */
325 cleanup_aer_uncorrect_error_status(oct->pci_dev);
329 * liquidio_pcie_error_detected - called when PCI error is detected
330 * @pdev: Pointer to PCI device
331 * @state: The current pci connection state
333 * This function is called after a PCI bus error affecting
334 * this device has been detected.
336 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
337 pci_channel_state_t state)
339 struct octeon_device *oct = pci_get_drvdata(pdev);
341 /* Non-correctable Non-fatal errors */
342 if (state == pci_channel_io_normal) {
343 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
344 cleanup_aer_uncorrect_error_status(oct->pci_dev);
345 return PCI_ERS_RESULT_CAN_RECOVER;
348 /* Non-correctable Fatal errors */
349 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
352 /* Always return a DISCONNECT. There is no support for recovery but only
353 * for a clean shutdown.
355 return PCI_ERS_RESULT_DISCONNECT;
359 * liquidio_pcie_mmio_enabled - mmio handler
360 * @pdev: Pointer to PCI device
362 static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev __maybe_unused *pdev)
364 /* We should never hit this since we never ask for a reset for a Fatal
365 * Error. We always return DISCONNECT in io_error above.
366 * But play safe and return RECOVERED for now.
368 return PCI_ERS_RESULT_RECOVERED;
372 * liquidio_pcie_slot_reset - called after the pci bus has been reset.
373 * @pdev: Pointer to PCI device
375 * Restart the card from scratch, as if from a cold-boot. Implementation
376 * resembles the first-half of the octeon_resume routine.
378 static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev __maybe_unused *pdev)
380 /* We should never hit this since we never ask for a reset for a Fatal
381 * Error. We always return DISCONNECT in io_error above.
382 * But play safe and return RECOVERED for now.
384 return PCI_ERS_RESULT_RECOVERED;
388 * liquidio_pcie_resume - called when traffic can start flowing again.
389 * @pdev: Pointer to PCI device
391 * This callback is called when the error recovery driver tells us that
392 * its OK to resume normal operation. Implementation resembles the
393 * second-half of the octeon_resume routine.
395 static void liquidio_pcie_resume(struct pci_dev __maybe_unused *pdev)
397 /* Nothing to be done here. */
400 #define liquidio_suspend NULL
401 #define liquidio_resume NULL
403 /* For PCI-E Advanced Error Recovery (AER) Interface */
404 static const struct pci_error_handlers liquidio_err_handler = {
405 .error_detected = liquidio_pcie_error_detected,
406 .mmio_enabled = liquidio_pcie_mmio_enabled,
407 .slot_reset = liquidio_pcie_slot_reset,
408 .resume = liquidio_pcie_resume,
411 static const struct pci_device_id liquidio_pci_tbl[] = {
413 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
416 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
419 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
425 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
427 static SIMPLE_DEV_PM_OPS(liquidio_pm_ops, liquidio_suspend, liquidio_resume);
429 static struct pci_driver liquidio_pci_driver = {
431 .id_table = liquidio_pci_tbl,
432 .probe = liquidio_probe,
433 .remove = liquidio_remove,
434 .err_handler = &liquidio_err_handler, /* For AER */
435 .driver.pm = &liquidio_pm_ops,
436 #ifdef CONFIG_PCI_IOV
437 .sriov_configure = liquidio_enable_sriov,
442 * liquidio_init_pci - register PCI driver
444 static int liquidio_init_pci(void)
446 return pci_register_driver(&liquidio_pci_driver);
450 * liquidio_deinit_pci - unregister PCI driver
452 static void liquidio_deinit_pci(void)
454 pci_unregister_driver(&liquidio_pci_driver);
458 * check_txq_status - Check Tx queue status, and take appropriate action
459 * @lio: per-network private data
460 * Return: 0 if full, number of queues woken up otherwise
462 static inline int check_txq_status(struct lio *lio)
464 int numqs = lio->netdev->real_num_tx_queues;
468 /* check each sub-queue state */
469 for (q = 0; q < numqs; q++) {
470 iq = lio->linfo.txpciq[q %
471 lio->oct_dev->num_iqs].s.q_no;
472 if (octnet_iq_is_full(lio->oct_dev, iq))
474 if (__netif_subqueue_stopped(lio->netdev, q)) {
475 netif_wake_subqueue(lio->netdev, q);
476 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
486 * print_link_info - Print link information
487 * @netdev: network device
489 static void print_link_info(struct net_device *netdev)
491 struct lio *lio = GET_LIO(netdev);
493 if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
494 ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
495 struct oct_link_info *linfo = &lio->linfo;
497 if (linfo->link.s.link_up) {
498 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
500 (linfo->link.s.duplex) ? "Full" : "Half");
502 netif_info(lio, link, lio->netdev, "Link Down\n");
508 * octnet_link_status_change - Routine to notify MTU change
509 * @work: work_struct data structure
511 static void octnet_link_status_change(struct work_struct *work)
513 struct cavium_wk *wk = (struct cavium_wk *)work;
514 struct lio *lio = (struct lio *)wk->ctxptr;
516 /* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
517 * this API is invoked only when new max-MTU of the interface is
518 * less than current MTU.
521 dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
526 * setup_link_status_change_wq - Sets up the mtu status change work
527 * @netdev: network device
529 static inline int setup_link_status_change_wq(struct net_device *netdev)
531 struct lio *lio = GET_LIO(netdev);
532 struct octeon_device *oct = lio->oct_dev;
534 lio->link_status_wq.wq = alloc_workqueue("link-status",
536 if (!lio->link_status_wq.wq) {
537 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
540 INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
541 octnet_link_status_change);
542 lio->link_status_wq.wk.ctxptr = lio;
547 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
549 struct lio *lio = GET_LIO(netdev);
551 if (lio->link_status_wq.wq) {
552 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
553 destroy_workqueue(lio->link_status_wq.wq);
558 * update_link_status - Update link status
559 * @netdev: network device
560 * @ls: link status structure
562 * Called on receipt of a link status response from the core application to
563 * update each interface's link status.
565 static inline void update_link_status(struct net_device *netdev,
566 union oct_link_status *ls)
568 struct lio *lio = GET_LIO(netdev);
569 int changed = (lio->linfo.link.u64 != ls->u64);
570 int current_max_mtu = lio->linfo.link.s.mtu;
571 struct octeon_device *oct = lio->oct_dev;
573 dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
574 __func__, lio->linfo.link.u64, ls->u64);
575 lio->linfo.link.u64 = ls->u64;
577 if ((lio->intf_open) && (changed)) {
578 print_link_info(netdev);
581 if (lio->linfo.link.s.link_up) {
582 dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
583 netif_carrier_on(netdev);
586 dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
587 netif_carrier_off(netdev);
590 if (lio->linfo.link.s.mtu != current_max_mtu) {
591 netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
592 current_max_mtu, lio->linfo.link.s.mtu);
593 netdev->max_mtu = lio->linfo.link.s.mtu;
595 if (lio->linfo.link.s.mtu < netdev->mtu) {
596 dev_warn(&oct->pci_dev->dev,
597 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
598 netdev->mtu, lio->linfo.link.s.mtu);
599 queue_delayed_work(lio->link_status_wq.wq,
600 &lio->link_status_wq.wk.work, 0);
606 * lio_sync_octeon_time - send latest localtime to octeon firmware so that
607 * firmware will correct it's time, in case there is a time skew
609 * @work: work scheduled to send time update to octeon firmware
611 static void lio_sync_octeon_time(struct work_struct *work)
613 struct cavium_wk *wk = (struct cavium_wk *)work;
614 struct lio *lio = (struct lio *)wk->ctxptr;
615 struct octeon_device *oct = lio->oct_dev;
616 struct octeon_soft_command *sc;
617 struct timespec64 ts;
621 sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
623 dev_err(&oct->pci_dev->dev,
624 "Failed to sync time to octeon: soft command allocation failed\n");
628 lt = (struct lio_time *)sc->virtdptr;
630 /* Get time of the day */
631 ktime_get_real_ts64(&ts);
633 lt->nsec = ts.tv_nsec;
634 octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
636 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
637 octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
638 OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
640 init_completion(&sc->complete);
641 sc->sc_status = OCTEON_REQUEST_PENDING;
643 ret = octeon_send_soft_command(oct, sc);
644 if (ret == IQ_SEND_FAILED) {
645 dev_err(&oct->pci_dev->dev,
646 "Failed to sync time to octeon: failed to send soft command\n");
647 octeon_free_soft_command(oct, sc);
649 WRITE_ONCE(sc->caller_is_done, true);
652 queue_delayed_work(lio->sync_octeon_time_wq.wq,
653 &lio->sync_octeon_time_wq.wk.work,
654 msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
658 * setup_sync_octeon_time_wq - prepare work to periodically update local time to octeon firmware
660 * @netdev: network device which should send time update to firmware
662 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
664 struct lio *lio = GET_LIO(netdev);
665 struct octeon_device *oct = lio->oct_dev;
667 lio->sync_octeon_time_wq.wq =
668 alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
669 if (!lio->sync_octeon_time_wq.wq) {
670 dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
673 INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
674 lio_sync_octeon_time);
675 lio->sync_octeon_time_wq.wk.ctxptr = lio;
676 queue_delayed_work(lio->sync_octeon_time_wq.wq,
677 &lio->sync_octeon_time_wq.wk.work,
678 msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
684 * cleanup_sync_octeon_time_wq - destroy wq
686 * @netdev: network device which should send time update to firmware
688 * Stop scheduling and destroy the work created to periodically update local
689 * time to octeon firmware.
691 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
693 struct lio *lio = GET_LIO(netdev);
694 struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
697 cancel_delayed_work_sync(&time_wq->wk.work);
698 destroy_workqueue(time_wq->wq);
702 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
704 struct octeon_device *other_oct;
706 other_oct = lio_get_device(oct->octeon_id + 1);
708 if (other_oct && other_oct->pci_dev) {
709 int oct_busnum, other_oct_busnum;
711 oct_busnum = oct->pci_dev->bus->number;
712 other_oct_busnum = other_oct->pci_dev->bus->number;
714 if (oct_busnum == other_oct_busnum) {
715 int oct_slot, other_oct_slot;
717 oct_slot = PCI_SLOT(oct->pci_dev->devfn);
718 other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
720 if (oct_slot == other_oct_slot)
728 static void disable_all_vf_links(struct octeon_device *oct)
730 struct net_device *netdev;
736 max_vfs = oct->sriov_info.max_vfs;
738 for (i = 0; i < oct->ifcount; i++) {
739 netdev = oct->props[i].netdev;
743 for (vf = 0; vf < max_vfs; vf++)
744 liquidio_set_vf_link_state(netdev, vf,
745 IFLA_VF_LINK_STATE_DISABLE);
749 static int liquidio_watchdog(void *param)
751 bool err_msg_was_printed[LIO_MAX_CORES];
752 u16 mask_of_crashed_or_stuck_cores = 0;
753 bool all_vf_links_are_disabled = false;
754 struct octeon_device *oct = param;
755 struct octeon_device *other_oct;
756 #ifdef CONFIG_MODULE_UNLOAD
757 long refcount, vfs_referencing_pf;
758 u64 vfs_mask1, vfs_mask2;
762 memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
764 while (!kthread_should_stop()) {
765 /* sleep for a couple of seconds so that we don't hog the CPU */
766 set_current_state(TASK_INTERRUPTIBLE);
767 schedule_timeout(msecs_to_jiffies(2000));
769 mask_of_crashed_or_stuck_cores =
770 (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
772 if (!mask_of_crashed_or_stuck_cores)
775 WRITE_ONCE(oct->cores_crashed, true);
776 other_oct = get_other_octeon_device(oct);
778 WRITE_ONCE(other_oct->cores_crashed, true);
780 for (core = 0; core < LIO_MAX_CORES; core++) {
781 bool core_crashed_or_got_stuck;
783 core_crashed_or_got_stuck =
784 (mask_of_crashed_or_stuck_cores
787 if (core_crashed_or_got_stuck &&
788 !err_msg_was_printed[core]) {
789 dev_err(&oct->pci_dev->dev,
790 "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n",
792 err_msg_was_printed[core] = true;
796 if (all_vf_links_are_disabled)
799 disable_all_vf_links(oct);
800 disable_all_vf_links(other_oct);
801 all_vf_links_are_disabled = true;
803 #ifdef CONFIG_MODULE_UNLOAD
804 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
805 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
807 vfs_referencing_pf = hweight64(vfs_mask1);
808 vfs_referencing_pf += hweight64(vfs_mask2);
810 refcount = module_refcount(THIS_MODULE);
811 if (refcount >= vfs_referencing_pf) {
812 while (vfs_referencing_pf) {
813 module_put(THIS_MODULE);
814 vfs_referencing_pf--;
824 * liquidio_probe - PCI probe handler
825 * @pdev: PCI device structure
829 liquidio_probe(struct pci_dev *pdev, const struct pci_device_id __maybe_unused *ent)
831 struct octeon_device *oct_dev = NULL;
832 struct handshake *hs;
834 oct_dev = octeon_allocate_device(pdev->device,
835 sizeof(struct octeon_device_priv));
837 dev_err(&pdev->dev, "Unable to allocate device\n");
841 if (pdev->device == OCTEON_CN23XX_PF_VID)
842 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
844 /* Enable PTP for 6XXX Device */
845 if (((pdev->device == OCTEON_CN66XX) ||
846 (pdev->device == OCTEON_CN68XX)))
847 oct_dev->ptp_enable = true;
849 oct_dev->ptp_enable = false;
851 dev_info(&pdev->dev, "Initializing device %x:%x.\n",
852 (u32)pdev->vendor, (u32)pdev->device);
854 /* Assign octeon_device for this device to the private data area. */
855 pci_set_drvdata(pdev, oct_dev);
857 /* set linux specific device pointer */
858 oct_dev->pci_dev = (void *)pdev;
860 oct_dev->subsystem_id = pdev->subsystem_vendor |
861 (pdev->subsystem_device << 16);
863 hs = &handshake[oct_dev->octeon_id];
864 init_completion(&hs->init);
865 init_completion(&hs->started);
868 if (oct_dev->octeon_id == 0)
869 /* first LiquidIO NIC is detected */
870 complete(&first_stage);
872 if (octeon_device_init(oct_dev)) {
874 liquidio_remove(pdev);
878 if (OCTEON_CN23XX_PF(oct_dev)) {
879 u8 bus, device, function;
881 if (atomic_read(oct_dev->adapter_refcount) == 1) {
882 /* Each NIC gets one watchdog kernel thread. The first
883 * PF (of each NIC) that gets pci_driver->probe()'d
884 * creates that thread.
886 bus = pdev->bus->number;
887 device = PCI_SLOT(pdev->devfn);
888 function = PCI_FUNC(pdev->devfn);
889 oct_dev->watchdog_task = kthread_run(liquidio_watchdog,
891 "liowd/%02hhx:%02hhx.%hhx",
892 bus, device, function);
893 if (IS_ERR(oct_dev->watchdog_task)) {
894 oct_dev->watchdog_task = NULL;
895 dev_err(&oct_dev->pci_dev->dev,
896 "failed to create kernel_thread\n");
897 liquidio_remove(pdev);
903 oct_dev->rx_pause = 1;
904 oct_dev->tx_pause = 1;
906 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
911 static bool fw_type_is_auto(void)
913 return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
914 sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
918 * octeon_pci_flr - PCI FLR for each Octeon device.
919 * @oct: octeon device
921 static void octeon_pci_flr(struct octeon_device *oct)
925 pci_save_state(oct->pci_dev);
927 pci_cfg_access_lock(oct->pci_dev);
929 /* Quiesce the device completely */
930 pci_write_config_word(oct->pci_dev, PCI_COMMAND,
931 PCI_COMMAND_INTX_DISABLE);
933 rc = __pci_reset_function_locked(oct->pci_dev);
936 dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
939 pci_cfg_access_unlock(oct->pci_dev);
941 pci_restore_state(oct->pci_dev);
945 * octeon_destroy_resources - Destroy resources associated with octeon device
946 * @oct: octeon device
948 static void octeon_destroy_resources(struct octeon_device *oct)
951 struct msix_entry *msix_entries;
952 struct octeon_device_priv *oct_priv = oct->priv;
954 struct handshake *hs;
956 switch (atomic_read(&oct->status)) {
957 case OCT_DEV_RUNNING:
958 case OCT_DEV_CORE_OK:
960 /* No more instructions will be forwarded. */
961 atomic_set(&oct->status, OCT_DEV_IN_RESET);
963 oct->app_mode = CVM_DRV_INVALID_APP;
964 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
965 lio_get_state_string(&oct->status));
967 schedule_timeout_uninterruptible(HZ / 10);
970 case OCT_DEV_HOST_OK:
972 case OCT_DEV_CONSOLE_INIT_DONE:
973 /* Remove any consoles */
974 octeon_remove_consoles(oct);
977 case OCT_DEV_IO_QUEUES_DONE:
978 if (lio_wait_for_instr_fetch(oct))
979 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
981 if (wait_for_pending_requests(oct))
982 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
984 /* Disable the input and output queues now. No more packets will
985 * arrive from Octeon, but we should wait for all packet
986 * processing to finish.
988 oct->fn_list.disable_io_queues(oct);
990 if (lio_wait_for_oq_pkts(oct))
991 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
993 /* Force all requests waiting to be fetched by OCTEON to
996 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
997 struct octeon_instr_queue *iq;
999 if (!(oct->io_qmask.iq & BIT_ULL(i)))
1001 iq = oct->instr_queue[i];
1003 if (atomic_read(&iq->instr_pending)) {
1004 spin_lock_bh(&iq->lock);
1006 iq->octeon_read_index = iq->host_write_index;
1007 iq->stats.instr_processed +=
1008 atomic_read(&iq->instr_pending);
1009 lio_process_iq_request_list(oct, iq, 0);
1010 spin_unlock_bh(&iq->lock);
1014 lio_process_ordered_list(oct, 1);
1015 octeon_free_sc_done_list(oct);
1016 octeon_free_sc_zombie_list(oct);
1019 case OCT_DEV_INTR_SET_DONE:
1020 /* Disable interrupts */
1021 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1024 msix_entries = (struct msix_entry *)oct->msix_entries;
1025 for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1026 if (oct->ioq_vector[i].vector) {
1027 /* clear the affinity_cpumask */
1028 irq_set_affinity_hint(
1029 msix_entries[i].vector,
1031 free_irq(msix_entries[i].vector,
1032 &oct->ioq_vector[i]);
1033 oct->ioq_vector[i].vector = 0;
1036 /* non-iov vector's argument is oct struct */
1037 free_irq(msix_entries[i].vector, oct);
1039 pci_disable_msix(oct->pci_dev);
1040 kfree(oct->msix_entries);
1041 oct->msix_entries = NULL;
1043 /* Release the interrupt line */
1044 free_irq(oct->pci_dev->irq, oct);
1046 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1047 pci_disable_msi(oct->pci_dev);
1050 kfree(oct->irq_name_storage);
1051 oct->irq_name_storage = NULL;
1054 case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1055 if (OCTEON_CN23XX_PF(oct))
1056 octeon_free_ioq_vector(oct);
1059 case OCT_DEV_MBOX_SETUP_DONE:
1060 if (OCTEON_CN23XX_PF(oct))
1061 oct->fn_list.free_mbox(oct);
1064 case OCT_DEV_IN_RESET:
1065 case OCT_DEV_DROQ_INIT_DONE:
1066 /* Wait for any pending operations */
1068 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1069 if (!(oct->io_qmask.oq & BIT_ULL(i)))
1071 octeon_delete_droq(oct, i);
1074 /* Force any pending handshakes to complete */
1075 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1079 handshake[oct->octeon_id].init_ok = 0;
1080 complete(&handshake[oct->octeon_id].init);
1081 handshake[oct->octeon_id].started_ok = 0;
1082 complete(&handshake[oct->octeon_id].started);
1087 case OCT_DEV_RESP_LIST_INIT_DONE:
1088 octeon_delete_response_list(oct);
1091 case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1092 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1093 if (!(oct->io_qmask.iq & BIT_ULL(i)))
1095 octeon_delete_instr_queue(oct, i);
1097 #ifdef CONFIG_PCI_IOV
1098 if (oct->sriov_info.sriov_enabled)
1099 pci_disable_sriov(oct->pci_dev);
1102 case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1103 octeon_free_sc_buffer_pool(oct);
1106 case OCT_DEV_DISPATCH_INIT_DONE:
1107 octeon_delete_dispatch_list(oct);
1108 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1111 case OCT_DEV_PCI_MAP_DONE:
1112 refcount = octeon_deregister_device(oct);
1114 /* Soft reset the octeon device before exiting.
1115 * However, if fw was loaded from card (i.e. autoboot),
1116 * perform an FLR instead.
1117 * Implementation note: only soft-reset the device
1118 * if it is a CN6XXX OR the LAST CN23XX device.
1120 if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1121 octeon_pci_flr(oct);
1122 else if (OCTEON_CN6XXX(oct) || !refcount)
1123 oct->fn_list.soft_reset(oct);
1125 octeon_unmap_pci_barx(oct, 0);
1126 octeon_unmap_pci_barx(oct, 1);
1129 case OCT_DEV_PCI_ENABLE_DONE:
1130 /* Disable the device, releasing the PCI INT */
1131 pci_disable_device(oct->pci_dev);
1134 case OCT_DEV_BEGIN_STATE:
1135 /* Nothing to be done here either */
1137 } /* end switch (oct->status) */
1139 tasklet_kill(&oct_priv->droq_tasklet);
1143 * send_rx_ctrl_cmd - Send Rx control command
1144 * @lio: per-network private data
1145 * @start_stop: whether to start or stop
1147 static int send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1149 struct octeon_soft_command *sc;
1150 union octnet_cmd *ncmd;
1151 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1154 if (oct->props[lio->ifidx].rx_on == start_stop)
1157 sc = (struct octeon_soft_command *)
1158 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1161 netif_info(lio, rx_err, lio->netdev,
1162 "Failed to allocate octeon_soft_command struct\n");
1166 ncmd = (union octnet_cmd *)sc->virtdptr;
1169 ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1170 ncmd->s.param1 = start_stop;
1172 octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1174 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1176 octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1177 OPCODE_NIC_CMD, 0, 0, 0);
1179 init_completion(&sc->complete);
1180 sc->sc_status = OCTEON_REQUEST_PENDING;
1182 retval = octeon_send_soft_command(oct, sc);
1183 if (retval == IQ_SEND_FAILED) {
1184 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1185 octeon_free_soft_command(oct, sc);
1187 /* Sleep on a wait queue till the cond flag indicates that the
1188 * response arrived or timed-out.
1190 retval = wait_for_sc_completion_timeout(oct, sc, 0);
1194 oct->props[lio->ifidx].rx_on = start_stop;
1195 WRITE_ONCE(sc->caller_is_done, true);
1202 * liquidio_destroy_nic_device - Destroy NIC device interface
1203 * @oct: octeon device
1204 * @ifidx: which interface to destroy
1206 * Cleanup associated with each interface for an Octeon device when NIC
1207 * module is being unloaded or if initialization fails during load.
1209 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1211 struct net_device *netdev = oct->props[ifidx].netdev;
1212 struct octeon_device_priv *oct_priv = oct->priv;
1213 struct napi_struct *napi, *n;
1217 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1222 lio = GET_LIO(netdev);
1224 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1226 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1227 liquidio_stop(netdev);
1229 if (oct->props[lio->ifidx].napi_enabled == 1) {
1230 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1233 oct->props[lio->ifidx].napi_enabled = 0;
1235 if (OCTEON_CN23XX_PF(oct))
1236 oct->droq[0]->ops.poll_mode = 0;
1240 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1241 netif_napi_del(napi);
1243 tasklet_enable(&oct_priv->droq_tasklet);
1245 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1246 unregister_netdev(netdev);
1248 cleanup_sync_octeon_time_wq(netdev);
1249 cleanup_link_status_change_wq(netdev);
1251 cleanup_rx_oom_poll_fn(netdev);
1253 lio_delete_glists(lio);
1255 free_netdev(netdev);
1257 oct->props[ifidx].gmxport = -1;
1259 oct->props[ifidx].netdev = NULL;
1263 * liquidio_stop_nic_module - Stop complete NIC functionality
1264 * @oct: octeon device
1266 static int liquidio_stop_nic_module(struct octeon_device *oct)
1271 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1272 device_lock(&oct->pci_dev->dev);
1274 devlink_unregister(oct->devlink);
1275 devlink_free(oct->devlink);
1276 oct->devlink = NULL;
1278 device_unlock(&oct->pci_dev->dev);
1280 if (!oct->ifcount) {
1281 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1285 spin_lock_bh(&oct->cmd_resp_wqlock);
1286 oct->cmd_resp_state = OCT_DRV_OFFLINE;
1287 spin_unlock_bh(&oct->cmd_resp_wqlock);
1289 lio_vf_rep_destroy(oct);
1291 for (i = 0; i < oct->ifcount; i++) {
1292 lio = GET_LIO(oct->props[i].netdev);
1293 for (j = 0; j < oct->num_oqs; j++)
1294 octeon_unregister_droq_ops(oct,
1295 lio->linfo.rxpciq[j].s.q_no);
1298 for (i = 0; i < oct->ifcount; i++)
1299 liquidio_destroy_nic_device(oct, i);
1301 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1306 * liquidio_remove - Cleans up resources at unload time
1307 * @pdev: PCI device structure
1309 static void liquidio_remove(struct pci_dev *pdev)
1311 struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1313 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1315 if (oct_dev->watchdog_task)
1316 kthread_stop(oct_dev->watchdog_task);
1318 if (!oct_dev->octeon_id &&
1319 oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1320 lio_vf_rep_modexit();
1322 if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1323 liquidio_stop_nic_module(oct_dev);
1325 /* Reset the octeon device and cleanup all memory allocated for
1326 * the octeon device by driver.
1328 octeon_destroy_resources(oct_dev);
1330 dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1332 /* This octeon device has been removed. Update the global
1333 * data structure to reflect this. Free the device structure.
1335 octeon_free_device_mem(oct_dev);
1339 * octeon_chip_specific_setup - Identify the Octeon device and to map the BAR address space
1340 * @oct: octeon device
1342 static int octeon_chip_specific_setup(struct octeon_device *oct)
1347 pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1348 pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1349 oct->rev_id = rev_id & 0xff;
1352 case OCTEON_CN68XX_PCIID:
1353 oct->chip_id = OCTEON_CN68XX;
1354 ret = lio_setup_cn68xx_octeon_device(oct);
1357 case OCTEON_CN66XX_PCIID:
1358 oct->chip_id = OCTEON_CN66XX;
1359 ret = lio_setup_cn66xx_octeon_device(oct);
1362 case OCTEON_CN23XX_PCIID_PF:
1363 oct->chip_id = OCTEON_CN23XX_PF_VID;
1364 ret = setup_cn23xx_octeon_pf_device(oct);
1367 #ifdef CONFIG_PCI_IOV
1369 pci_sriov_set_totalvfs(oct->pci_dev,
1370 oct->sriov_info.max_vfs);
1375 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1383 * octeon_pci_os_setup - PCI initialization for each Octeon device.
1384 * @oct: octeon device
1386 static int octeon_pci_os_setup(struct octeon_device *oct)
1388 /* setup PCI stuff first */
1389 if (pci_enable_device(oct->pci_dev)) {
1390 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1394 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1395 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1396 pci_disable_device(oct->pci_dev);
1400 /* Enable PCI DMA Master. */
1401 pci_set_master(oct->pci_dev);
1407 * free_netbuf - Unmap and free network buffer
1410 static void free_netbuf(void *buf)
1412 struct sk_buff *skb;
1413 struct octnet_buf_free_info *finfo;
1416 finfo = (struct octnet_buf_free_info *)buf;
1420 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1423 tx_buffer_free(skb);
1427 * free_netsgbuf - Unmap and free gather buffer
1430 static void free_netsgbuf(void *buf)
1432 struct octnet_buf_free_info *finfo;
1433 struct sk_buff *skb;
1435 struct octnic_gather *g;
1438 finfo = (struct octnet_buf_free_info *)buf;
1442 frags = skb_shinfo(skb)->nr_frags;
1444 dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1445 g->sg[0].ptr[0], (skb->len - skb->data_len),
1450 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1452 dma_unmap_page(&lio->oct_dev->pci_dev->dev,
1453 g->sg[(i >> 2)].ptr[(i & 3)],
1454 skb_frag_size(frag), DMA_TO_DEVICE);
1458 iq = skb_iq(lio->oct_dev, skb);
1459 spin_lock(&lio->glist_lock[iq]);
1460 list_add_tail(&g->list, &lio->glist[iq]);
1461 spin_unlock(&lio->glist_lock[iq]);
1463 tx_buffer_free(skb);
1467 * free_netsgbuf_with_resp - Unmap and free gather buffer with response
1470 static void free_netsgbuf_with_resp(void *buf)
1472 struct octeon_soft_command *sc;
1473 struct octnet_buf_free_info *finfo;
1474 struct sk_buff *skb;
1476 struct octnic_gather *g;
1479 sc = (struct octeon_soft_command *)buf;
1480 skb = (struct sk_buff *)sc->callback_arg;
1481 finfo = (struct octnet_buf_free_info *)&skb->cb;
1485 frags = skb_shinfo(skb)->nr_frags;
1487 dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1488 g->sg[0].ptr[0], (skb->len - skb->data_len),
1493 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1495 dma_unmap_page(&lio->oct_dev->pci_dev->dev,
1496 g->sg[(i >> 2)].ptr[(i & 3)],
1497 skb_frag_size(frag), DMA_TO_DEVICE);
1501 iq = skb_iq(lio->oct_dev, skb);
1503 spin_lock(&lio->glist_lock[iq]);
1504 list_add_tail(&g->list, &lio->glist[iq]);
1505 spin_unlock(&lio->glist_lock[iq]);
1507 /* Don't free the skb yet */
1511 * liquidio_ptp_adjfine - Adjust ptp frequency
1512 * @ptp: PTP clock info
1513 * @scaled_ppm: how much to adjust by, in scaled parts-per-million
1515 * Scaled parts per million is ppm with a 16-bit binary fractional field.
1517 static int liquidio_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
1519 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1520 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1521 s32 ppb = scaled_ppm_to_ppb(scaled_ppm);
1523 unsigned long flags;
1524 bool neg_adj = false;
1531 /* The hardware adds the clock compensation value to the
1532 * PTP clock on every coprocessor clock cycle, so we
1533 * compute the delta in terms of coprocessor clocks.
1535 delta = (u64)ppb << 32;
1536 do_div(delta, oct->coproc_clock_rate);
1538 spin_lock_irqsave(&lio->ptp_lock, flags);
1539 comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1544 lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1545 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1551 * liquidio_ptp_adjtime - Adjust ptp time
1552 * @ptp: PTP clock info
1553 * @delta: how much to adjust by, in nanosecs
1555 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1557 unsigned long flags;
1558 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1560 spin_lock_irqsave(&lio->ptp_lock, flags);
1561 lio->ptp_adjust += delta;
1562 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1568 * liquidio_ptp_gettime - Get hardware clock time, including any adjustment
1569 * @ptp: PTP clock info
1572 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1573 struct timespec64 *ts)
1576 unsigned long flags;
1577 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1578 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1580 spin_lock_irqsave(&lio->ptp_lock, flags);
1581 ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1582 ns += lio->ptp_adjust;
1583 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1585 *ts = ns_to_timespec64(ns);
1591 * liquidio_ptp_settime - Set hardware clock time. Reset adjustment
1592 * @ptp: PTP clock info
1595 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1596 const struct timespec64 *ts)
1599 unsigned long flags;
1600 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1601 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1603 ns = timespec64_to_ns(ts);
1605 spin_lock_irqsave(&lio->ptp_lock, flags);
1606 lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1607 lio->ptp_adjust = 0;
1608 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1614 * liquidio_ptp_enable - Check if PTP is enabled
1615 * @ptp: PTP clock info
1620 liquidio_ptp_enable(struct ptp_clock_info __maybe_unused *ptp,
1621 struct ptp_clock_request __maybe_unused *rq,
1622 int __maybe_unused on)
1628 * oct_ptp_open - Open PTP clock source
1629 * @netdev: network device
1631 static void oct_ptp_open(struct net_device *netdev)
1633 struct lio *lio = GET_LIO(netdev);
1634 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1636 spin_lock_init(&lio->ptp_lock);
1638 snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1639 lio->ptp_info.owner = THIS_MODULE;
1640 lio->ptp_info.max_adj = 250000000;
1641 lio->ptp_info.n_alarm = 0;
1642 lio->ptp_info.n_ext_ts = 0;
1643 lio->ptp_info.n_per_out = 0;
1644 lio->ptp_info.pps = 0;
1645 lio->ptp_info.adjfine = liquidio_ptp_adjfine;
1646 lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1647 lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1648 lio->ptp_info.settime64 = liquidio_ptp_settime;
1649 lio->ptp_info.enable = liquidio_ptp_enable;
1651 lio->ptp_adjust = 0;
1653 lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1654 &oct->pci_dev->dev);
1656 if (IS_ERR(lio->ptp_clock))
1657 lio->ptp_clock = NULL;
1661 * liquidio_ptp_init - Init PTP clock
1662 * @oct: octeon device
1664 static void liquidio_ptp_init(struct octeon_device *oct)
1666 u64 clock_comp, cfg;
1668 clock_comp = (u64)NSEC_PER_SEC << 32;
1669 do_div(clock_comp, oct->coproc_clock_rate);
1670 lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1673 cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1674 lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1678 * load_firmware - Load firmware to device
1679 * @oct: octeon device
1681 * Maps device to firmware filename, requests firmware, and downloads it
1683 static int load_firmware(struct octeon_device *oct)
1686 const struct firmware *fw;
1687 char fw_name[LIO_MAX_FW_FILENAME_LEN];
1690 if (fw_type_is_auto()) {
1691 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1692 strscpy_pad(fw_type, tmp_fw_type, sizeof(fw_type));
1694 tmp_fw_type = fw_type;
1697 sprintf(fw_name, "/*(DEBLOBBED)*/", LIO_FW_DIR, LIO_FW_BASE_NAME,
1698 octeon_get_conf(oct)->card_name, tmp_fw_type,
1699 LIO_FW_NAME_SUFFIX);
1701 ret = reject_firmware(&fw, fw_name, &oct->pci_dev->dev);
1703 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1705 release_firmware(fw);
1709 ret = octeon_download_firmware(oct, fw->data, fw->size);
1711 release_firmware(fw);
1717 * octnet_poll_check_txq_status - Poll routine for checking transmit queue status
1718 * @work: work_struct data structure
1720 static void octnet_poll_check_txq_status(struct work_struct *work)
1722 struct cavium_wk *wk = (struct cavium_wk *)work;
1723 struct lio *lio = (struct lio *)wk->ctxptr;
1725 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1728 check_txq_status(lio);
1729 queue_delayed_work(lio->txq_status_wq.wq,
1730 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1734 * setup_tx_poll_fn - Sets up the txq poll check
1735 * @netdev: network device
1737 static inline int setup_tx_poll_fn(struct net_device *netdev)
1739 struct lio *lio = GET_LIO(netdev);
1740 struct octeon_device *oct = lio->oct_dev;
1742 lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1744 if (!lio->txq_status_wq.wq) {
1745 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1748 INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1749 octnet_poll_check_txq_status);
1750 lio->txq_status_wq.wk.ctxptr = lio;
1751 queue_delayed_work(lio->txq_status_wq.wq,
1752 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1756 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1758 struct lio *lio = GET_LIO(netdev);
1760 if (lio->txq_status_wq.wq) {
1761 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1762 destroy_workqueue(lio->txq_status_wq.wq);
1767 * liquidio_open - Net device open for LiquidIO
1768 * @netdev: network device
1770 static int liquidio_open(struct net_device *netdev)
1772 struct lio *lio = GET_LIO(netdev);
1773 struct octeon_device *oct = lio->oct_dev;
1774 struct octeon_device_priv *oct_priv = oct->priv;
1775 struct napi_struct *napi, *n;
1778 if (oct->props[lio->ifidx].napi_enabled == 0) {
1779 tasklet_disable(&oct_priv->droq_tasklet);
1781 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1784 oct->props[lio->ifidx].napi_enabled = 1;
1786 if (OCTEON_CN23XX_PF(oct))
1787 oct->droq[0]->ops.poll_mode = 1;
1790 if (oct->ptp_enable)
1791 oct_ptp_open(netdev);
1793 ifstate_set(lio, LIO_IFSTATE_RUNNING);
1795 if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on) {
1796 ret = setup_tx_poll_fn(netdev);
1801 netif_tx_start_all_queues(netdev);
1803 /* Ready for link status updates */
1806 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1808 /* tell Octeon to start forwarding packets to host */
1809 ret = send_rx_ctrl_cmd(lio, 1);
1813 /* start periodical statistics fetch */
1814 INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1815 lio->stats_wk.ctxptr = lio;
1816 schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1817 (LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1819 dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1825 if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on)
1826 cleanup_tx_poll_fn(netdev);
1828 if (lio->ptp_clock) {
1829 ptp_clock_unregister(lio->ptp_clock);
1830 lio->ptp_clock = NULL;
1833 if (oct->props[lio->ifidx].napi_enabled == 1) {
1834 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1837 oct->props[lio->ifidx].napi_enabled = 0;
1839 if (OCTEON_CN23XX_PF(oct))
1840 oct->droq[0]->ops.poll_mode = 0;
1847 * liquidio_stop - Net device stop for LiquidIO
1848 * @netdev: network device
1850 static int liquidio_stop(struct net_device *netdev)
1852 struct lio *lio = GET_LIO(netdev);
1853 struct octeon_device *oct = lio->oct_dev;
1854 struct octeon_device_priv *oct_priv = oct->priv;
1855 struct napi_struct *napi, *n;
1858 ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1860 /* Stop any link updates */
1865 /* Inform that netif carrier is down */
1866 netif_carrier_off(netdev);
1867 netif_tx_disable(netdev);
1869 lio->linfo.link.s.link_up = 0;
1870 lio->link_changes++;
1872 /* Tell Octeon that nic interface is down. */
1873 ret = send_rx_ctrl_cmd(lio, 0);
1877 if (OCTEON_CN23XX_PF(oct)) {
1879 cleanup_tx_poll_fn(netdev);
1881 cleanup_tx_poll_fn(netdev);
1884 cancel_delayed_work_sync(&lio->stats_wk.work);
1886 if (lio->ptp_clock) {
1887 ptp_clock_unregister(lio->ptp_clock);
1888 lio->ptp_clock = NULL;
1891 /* Wait for any pending Rx descriptors */
1892 if (lio_wait_for_clean_oq(oct))
1893 netif_info(lio, rx_err, lio->netdev,
1894 "Proceeding with stop interface after partial RX desc processing\n");
1896 if (oct->props[lio->ifidx].napi_enabled == 1) {
1897 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1900 oct->props[lio->ifidx].napi_enabled = 0;
1902 if (OCTEON_CN23XX_PF(oct))
1903 oct->droq[0]->ops.poll_mode = 0;
1905 tasklet_enable(&oct_priv->droq_tasklet);
1908 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1914 * get_new_flags - Converts a mask based on net device flags
1915 * @netdev: network device
1917 * This routine generates a octnet_ifflags mask from the net device flags
1918 * received from the OS.
1920 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1922 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1924 if (netdev->flags & IFF_PROMISC)
1925 f |= OCTNET_IFFLAG_PROMISC;
1927 if (netdev->flags & IFF_ALLMULTI)
1928 f |= OCTNET_IFFLAG_ALLMULTI;
1930 if (netdev->flags & IFF_MULTICAST) {
1931 f |= OCTNET_IFFLAG_MULTICAST;
1933 /* Accept all multicast addresses if there are more than we
1936 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1937 f |= OCTNET_IFFLAG_ALLMULTI;
1940 if (netdev->flags & IFF_BROADCAST)
1941 f |= OCTNET_IFFLAG_BROADCAST;
1947 * liquidio_set_mcast_list - Net device set_multicast_list
1948 * @netdev: network device
1950 static void liquidio_set_mcast_list(struct net_device *netdev)
1952 struct lio *lio = GET_LIO(netdev);
1953 struct octeon_device *oct = lio->oct_dev;
1954 struct octnic_ctrl_pkt nctrl;
1955 struct netdev_hw_addr *ha;
1958 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1960 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1962 /* Create a ctrl pkt command to be sent to core app. */
1964 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1965 nctrl.ncmd.s.param1 = get_new_flags(netdev);
1966 nctrl.ncmd.s.param2 = mc_count;
1967 nctrl.ncmd.s.more = mc_count;
1968 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1969 nctrl.netpndev = (u64)netdev;
1970 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1972 /* copy all the addresses into the udd */
1974 netdev_for_each_mc_addr(ha, netdev) {
1976 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1977 /* no need to swap bytes */
1979 if (++mc > &nctrl.udd[mc_count])
1983 /* Apparently, any activity in this call from the kernel has to
1984 * be atomic. So we won't wait for response.
1987 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1989 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
1995 * liquidio_set_mac - Net device set_mac_address
1996 * @netdev: network device
1997 * @p: pointer to sockaddr
1999 static int liquidio_set_mac(struct net_device *netdev, void *p)
2002 struct lio *lio = GET_LIO(netdev);
2003 struct octeon_device *oct = lio->oct_dev;
2004 struct sockaddr *addr = (struct sockaddr *)p;
2005 struct octnic_ctrl_pkt nctrl;
2007 if (!is_valid_ether_addr(addr->sa_data))
2008 return -EADDRNOTAVAIL;
2010 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2013 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2014 nctrl.ncmd.s.param1 = 0;
2015 nctrl.ncmd.s.more = 1;
2016 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2017 nctrl.netpndev = (u64)netdev;
2020 /* The MAC Address is presented in network byte order. */
2021 memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2023 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2025 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2029 if (nctrl.sc_status) {
2030 dev_err(&oct->pci_dev->dev,
2031 "%s: MAC Address change failed. sc return=%x\n",
2032 __func__, nctrl.sc_status);
2036 eth_hw_addr_set(netdev, addr->sa_data);
2037 memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2043 liquidio_get_stats64(struct net_device *netdev,
2044 struct rtnl_link_stats64 *lstats)
2046 struct lio *lio = GET_LIO(netdev);
2047 struct octeon_device *oct;
2048 u64 pkts = 0, drop = 0, bytes = 0;
2049 struct oct_droq_stats *oq_stats;
2050 struct oct_iq_stats *iq_stats;
2051 int i, iq_no, oq_no;
2055 if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2058 for (i = 0; i < oct->num_iqs; i++) {
2059 iq_no = lio->linfo.txpciq[i].s.q_no;
2060 iq_stats = &oct->instr_queue[iq_no]->stats;
2061 pkts += iq_stats->tx_done;
2062 drop += iq_stats->tx_dropped;
2063 bytes += iq_stats->tx_tot_bytes;
2066 lstats->tx_packets = pkts;
2067 lstats->tx_bytes = bytes;
2068 lstats->tx_dropped = drop;
2074 for (i = 0; i < oct->num_oqs; i++) {
2075 oq_no = lio->linfo.rxpciq[i].s.q_no;
2076 oq_stats = &oct->droq[oq_no]->stats;
2077 pkts += oq_stats->rx_pkts_received;
2078 drop += (oq_stats->rx_dropped +
2079 oq_stats->dropped_nodispatch +
2080 oq_stats->dropped_toomany +
2081 oq_stats->dropped_nomem);
2082 bytes += oq_stats->rx_bytes_received;
2085 lstats->rx_bytes = bytes;
2086 lstats->rx_packets = pkts;
2087 lstats->rx_dropped = drop;
2089 lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2090 lstats->collisions = oct->link_stats.fromhost.total_collisions;
2092 /* detailed rx_errors: */
2093 lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2094 /* recved pkt with crc error */
2095 lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2096 /* recv'd frame alignment error */
2097 lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2098 /* recv'r fifo overrun */
2099 lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2101 lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2102 lstats->rx_frame_errors + lstats->rx_fifo_errors;
2104 /* detailed tx_errors */
2105 lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2106 lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2107 lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2109 lstats->tx_errors = lstats->tx_aborted_errors +
2110 lstats->tx_carrier_errors +
2111 lstats->tx_fifo_errors;
2115 * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl
2116 * @netdev: network device
2117 * @ifr: interface request
2119 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2121 struct hwtstamp_config conf;
2122 struct lio *lio = GET_LIO(netdev);
2124 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2127 switch (conf.tx_type) {
2128 case HWTSTAMP_TX_ON:
2129 case HWTSTAMP_TX_OFF:
2135 switch (conf.rx_filter) {
2136 case HWTSTAMP_FILTER_NONE:
2138 case HWTSTAMP_FILTER_ALL:
2139 case HWTSTAMP_FILTER_SOME:
2140 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2141 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2142 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2143 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2144 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2145 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2146 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2147 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2148 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2149 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2150 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2151 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2152 case HWTSTAMP_FILTER_NTP_ALL:
2153 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2159 if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2160 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2163 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2165 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2169 * liquidio_ioctl - ioctl handler
2170 * @netdev: network device
2171 * @ifr: interface request
2174 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2176 struct lio *lio = GET_LIO(netdev);
2180 if (lio->oct_dev->ptp_enable)
2181 return hwtstamp_ioctl(netdev, ifr);
2189 * handle_timestamp - handle a Tx timestamp response
2190 * @oct: octeon device
2191 * @status: response status
2192 * @buf: pointer to skb
2194 static void handle_timestamp(struct octeon_device *oct,
2198 struct octnet_buf_free_info *finfo;
2199 struct octeon_soft_command *sc;
2200 struct oct_timestamp_resp *resp;
2202 struct sk_buff *skb = (struct sk_buff *)buf;
2204 finfo = (struct octnet_buf_free_info *)skb->cb;
2208 resp = (struct oct_timestamp_resp *)sc->virtrptr;
2210 if (status != OCTEON_REQUEST_DONE) {
2211 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2212 CVM_CAST64(status));
2213 resp->timestamp = 0;
2216 octeon_swap_8B_data(&resp->timestamp, 1);
2218 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2219 struct skb_shared_hwtstamps ts;
2220 u64 ns = resp->timestamp;
2222 netif_info(lio, tx_done, lio->netdev,
2223 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2224 skb, (unsigned long long)ns);
2225 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2226 skb_tstamp_tx(skb, &ts);
2229 octeon_free_soft_command(oct, sc);
2230 tx_buffer_free(skb);
2234 * send_nic_timestamp_pkt - Send a data packet that will be timestamped
2235 * @oct: octeon device
2236 * @ndata: pointer to network data
2237 * @finfo: pointer to private network data
2238 * @xmit_more: more is coming
2240 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2241 struct octnic_data_pkt *ndata,
2242 struct octnet_buf_free_info *finfo,
2246 struct octeon_soft_command *sc;
2253 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2254 sizeof(struct oct_timestamp_resp));
2258 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2259 return IQ_SEND_FAILED;
2262 if (ndata->reqtype == REQTYPE_NORESP_NET)
2263 ndata->reqtype = REQTYPE_RESP_NET;
2264 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2265 ndata->reqtype = REQTYPE_RESP_NET_SG;
2267 sc->callback = handle_timestamp;
2268 sc->callback_arg = finfo->skb;
2269 sc->iq_no = ndata->q_no;
2271 if (OCTEON_CN23XX_PF(oct))
2272 len = (u32)((struct octeon_instr_ih3 *)
2273 (&sc->cmd.cmd3.ih3))->dlengsz;
2275 len = (u32)((struct octeon_instr_ih2 *)
2276 (&sc->cmd.cmd2.ih2))->dlengsz;
2278 ring_doorbell = !xmit_more;
2280 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2281 sc, len, ndata->reqtype);
2283 if (retval == IQ_SEND_FAILED) {
2284 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2286 octeon_free_soft_command(oct, sc);
2288 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2295 * liquidio_xmit - Transmit networks packets to the Octeon interface
2296 * @skb: skbuff struct to be passed to network layer.
2297 * @netdev: pointer to network device
2299 * Return: whether the packet was transmitted to the device okay or not
2300 * (NETDEV_TX_OK or NETDEV_TX_BUSY)
2302 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2305 struct octnet_buf_free_info *finfo;
2306 union octnic_cmd_setup cmdsetup;
2307 struct octnic_data_pkt ndata;
2308 struct octeon_device *oct;
2309 struct oct_iq_stats *stats;
2310 struct octeon_instr_irh *irh;
2311 union tx_info *tx_info;
2313 int q_idx = 0, iq_no = 0;
2314 int j, xmit_more = 0;
2318 lio = GET_LIO(netdev);
2321 q_idx = skb_iq(oct, skb);
2323 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2325 stats = &oct->instr_queue[iq_no]->stats;
2327 /* Check for all conditions in which the current packet cannot be
2330 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2331 (!lio->linfo.link.s.link_up) ||
2333 netif_info(lio, tx_err, lio->netdev,
2334 "Transmit failed link_status : %d\n",
2335 lio->linfo.link.s.link_up);
2336 goto lio_xmit_failed;
2339 /* Use space in skb->cb to store info used to unmap and
2342 finfo = (struct octnet_buf_free_info *)skb->cb;
2347 /* Prepare the attributes for the data to be passed to OSI. */
2348 memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2350 ndata.buf = (void *)finfo;
2354 if (octnet_iq_is_full(oct, ndata.q_no)) {
2355 /* defer sending if queue is full */
2356 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2358 stats->tx_iq_busy++;
2359 return NETDEV_TX_BUSY;
2362 /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n",
2363 * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2366 ndata.datasize = skb->len;
2369 cmdsetup.s.iq_no = iq_no;
2371 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2372 if (skb->encapsulation) {
2373 cmdsetup.s.tnl_csum = 1;
2376 cmdsetup.s.transport_csum = 1;
2379 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2380 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2381 cmdsetup.s.timestamp = 1;
2384 if (skb_shinfo(skb)->nr_frags == 0) {
2385 cmdsetup.s.u.datasize = skb->len;
2386 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2388 /* Offload checksum calculation for TCP/UDP packets */
2389 dptr = dma_map_single(&oct->pci_dev->dev,
2393 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2394 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2396 stats->tx_dmamap_fail++;
2397 return NETDEV_TX_BUSY;
2400 if (OCTEON_CN23XX_PF(oct))
2401 ndata.cmd.cmd3.dptr = dptr;
2403 ndata.cmd.cmd2.dptr = dptr;
2405 ndata.reqtype = REQTYPE_NORESP_NET;
2410 struct octnic_gather *g;
2412 spin_lock(&lio->glist_lock[q_idx]);
2413 g = (struct octnic_gather *)
2414 lio_list_delete_head(&lio->glist[q_idx]);
2415 spin_unlock(&lio->glist_lock[q_idx]);
2418 netif_info(lio, tx_err, lio->netdev,
2419 "Transmit scatter gather: glist null!\n");
2420 goto lio_xmit_failed;
2423 cmdsetup.s.gather = 1;
2424 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2425 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2427 memset(g->sg, 0, g->sg_size);
2429 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2431 (skb->len - skb->data_len),
2433 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2434 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2436 stats->tx_dmamap_fail++;
2437 return NETDEV_TX_BUSY;
2439 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2441 frags = skb_shinfo(skb)->nr_frags;
2444 frag = &skb_shinfo(skb)->frags[i - 1];
2446 g->sg[(i >> 2)].ptr[(i & 3)] =
2447 skb_frag_dma_map(&oct->pci_dev->dev,
2448 frag, 0, skb_frag_size(frag),
2451 if (dma_mapping_error(&oct->pci_dev->dev,
2452 g->sg[i >> 2].ptr[i & 3])) {
2453 dma_unmap_single(&oct->pci_dev->dev,
2455 skb->len - skb->data_len,
2457 for (j = 1; j < i; j++) {
2458 frag = &skb_shinfo(skb)->frags[j - 1];
2459 dma_unmap_page(&oct->pci_dev->dev,
2460 g->sg[j >> 2].ptr[j & 3],
2461 skb_frag_size(frag),
2464 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2466 return NETDEV_TX_BUSY;
2469 add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
2474 dptr = g->sg_dma_ptr;
2476 if (OCTEON_CN23XX_PF(oct))
2477 ndata.cmd.cmd3.dptr = dptr;
2479 ndata.cmd.cmd2.dptr = dptr;
2483 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2486 if (OCTEON_CN23XX_PF(oct)) {
2487 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2488 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2490 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2491 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2494 if (skb_shinfo(skb)->gso_size) {
2495 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2496 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2500 /* HW insert VLAN tag */
2501 if (skb_vlan_tag_present(skb)) {
2502 irh->priority = skb_vlan_tag_get(skb) >> 13;
2503 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2506 xmit_more = netdev_xmit_more();
2508 if (unlikely(cmdsetup.s.timestamp))
2509 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2511 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2512 if (status == IQ_SEND_FAILED)
2513 goto lio_xmit_failed;
2515 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2517 if (status == IQ_SEND_STOP)
2518 netif_stop_subqueue(netdev, q_idx);
2520 netif_trans_update(netdev);
2522 if (tx_info->s.gso_segs)
2523 stats->tx_done += tx_info->s.gso_segs;
2526 stats->tx_tot_bytes += ndata.datasize;
2528 return NETDEV_TX_OK;
2531 stats->tx_dropped++;
2532 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2533 iq_no, stats->tx_dropped);
2535 dma_unmap_single(&oct->pci_dev->dev, dptr,
2536 ndata.datasize, DMA_TO_DEVICE);
2538 octeon_ring_doorbell_locked(oct, iq_no);
2540 tx_buffer_free(skb);
2541 return NETDEV_TX_OK;
2545 * liquidio_tx_timeout - Network device Tx timeout
2546 * @netdev: pointer to network device
2547 * @txqueue: index of the hung transmit queue
2549 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)
2553 lio = GET_LIO(netdev);
2555 netif_info(lio, tx_err, lio->netdev,
2556 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2557 netdev->stats.tx_dropped);
2558 netif_trans_update(netdev);
2562 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2563 __be16 proto __attribute__((unused)),
2566 struct lio *lio = GET_LIO(netdev);
2567 struct octeon_device *oct = lio->oct_dev;
2568 struct octnic_ctrl_pkt nctrl;
2571 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2574 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2575 nctrl.ncmd.s.param1 = vid;
2576 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2577 nctrl.netpndev = (u64)netdev;
2578 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2580 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2582 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2591 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2592 __be16 proto __attribute__((unused)),
2595 struct lio *lio = GET_LIO(netdev);
2596 struct octeon_device *oct = lio->oct_dev;
2597 struct octnic_ctrl_pkt nctrl;
2600 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2603 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2604 nctrl.ncmd.s.param1 = vid;
2605 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2606 nctrl.netpndev = (u64)netdev;
2607 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2609 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2611 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2620 * liquidio_set_rxcsum_command - Sending command to enable/disable RX checksum offload
2621 * @netdev: pointer to network device
2622 * @command: OCTNET_CMD_TNL_RX_CSUM_CTL
2623 * @rx_cmd: OCTNET_CMD_RXCSUM_ENABLE/OCTNET_CMD_RXCSUM_DISABLE
2624 * Returns: SUCCESS or FAILURE
2626 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2629 struct lio *lio = GET_LIO(netdev);
2630 struct octeon_device *oct = lio->oct_dev;
2631 struct octnic_ctrl_pkt nctrl;
2634 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2637 nctrl.ncmd.s.cmd = command;
2638 nctrl.ncmd.s.param1 = rx_cmd;
2639 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2640 nctrl.netpndev = (u64)netdev;
2641 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2643 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2645 dev_err(&oct->pci_dev->dev,
2646 "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2655 * liquidio_vxlan_port_command - Sending command to add/delete VxLAN UDP port to firmware
2656 * @netdev: pointer to network device
2657 * @command: OCTNET_CMD_VXLAN_PORT_CONFIG
2658 * @vxlan_port: VxLAN port to be added or deleted
2659 * @vxlan_cmd_bit: OCTNET_CMD_VXLAN_PORT_ADD,
2660 * OCTNET_CMD_VXLAN_PORT_DEL
2661 * Return: SUCCESS or FAILURE
2663 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2664 u16 vxlan_port, u8 vxlan_cmd_bit)
2666 struct lio *lio = GET_LIO(netdev);
2667 struct octeon_device *oct = lio->oct_dev;
2668 struct octnic_ctrl_pkt nctrl;
2671 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2674 nctrl.ncmd.s.cmd = command;
2675 nctrl.ncmd.s.more = vxlan_cmd_bit;
2676 nctrl.ncmd.s.param1 = vxlan_port;
2677 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2678 nctrl.netpndev = (u64)netdev;
2679 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2681 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2683 dev_err(&oct->pci_dev->dev,
2684 "VxLAN port add/delete failed in core (ret:0x%x)\n",
2692 static int liquidio_udp_tunnel_set_port(struct net_device *netdev,
2693 unsigned int table, unsigned int entry,
2694 struct udp_tunnel_info *ti)
2696 return liquidio_vxlan_port_command(netdev,
2697 OCTNET_CMD_VXLAN_PORT_CONFIG,
2699 OCTNET_CMD_VXLAN_PORT_ADD);
2702 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev,
2705 struct udp_tunnel_info *ti)
2707 return liquidio_vxlan_port_command(netdev,
2708 OCTNET_CMD_VXLAN_PORT_CONFIG,
2710 OCTNET_CMD_VXLAN_PORT_DEL);
2713 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = {
2714 .set_port = liquidio_udp_tunnel_set_port,
2715 .unset_port = liquidio_udp_tunnel_unset_port,
2717 { .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
2722 * liquidio_fix_features - Net device fix features
2723 * @netdev: pointer to network device
2724 * @request: features requested
2725 * Return: updated features list
2727 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2728 netdev_features_t request)
2730 struct lio *lio = netdev_priv(netdev);
2732 if ((request & NETIF_F_RXCSUM) &&
2733 !(lio->dev_capability & NETIF_F_RXCSUM))
2734 request &= ~NETIF_F_RXCSUM;
2736 if ((request & NETIF_F_HW_CSUM) &&
2737 !(lio->dev_capability & NETIF_F_HW_CSUM))
2738 request &= ~NETIF_F_HW_CSUM;
2740 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2741 request &= ~NETIF_F_TSO;
2743 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2744 request &= ~NETIF_F_TSO6;
2746 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2747 request &= ~NETIF_F_LRO;
2749 /*Disable LRO if RXCSUM is off */
2750 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2751 (lio->dev_capability & NETIF_F_LRO))
2752 request &= ~NETIF_F_LRO;
2754 if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2755 !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2756 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2762 * liquidio_set_features - Net device set features
2763 * @netdev: pointer to network device
2764 * @features: features to enable/disable
2766 static int liquidio_set_features(struct net_device *netdev,
2767 netdev_features_t features)
2769 struct lio *lio = netdev_priv(netdev);
2771 if ((features & NETIF_F_LRO) &&
2772 (lio->dev_capability & NETIF_F_LRO) &&
2773 !(netdev->features & NETIF_F_LRO))
2774 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2775 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2776 else if (!(features & NETIF_F_LRO) &&
2777 (lio->dev_capability & NETIF_F_LRO) &&
2778 (netdev->features & NETIF_F_LRO))
2779 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2780 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2782 /* Sending command to firmware to enable/disable RX checksum
2783 * offload settings using ethtool
2785 if (!(netdev->features & NETIF_F_RXCSUM) &&
2786 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2787 (features & NETIF_F_RXCSUM))
2788 liquidio_set_rxcsum_command(netdev,
2789 OCTNET_CMD_TNL_RX_CSUM_CTL,
2790 OCTNET_CMD_RXCSUM_ENABLE);
2791 else if ((netdev->features & NETIF_F_RXCSUM) &&
2792 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2793 !(features & NETIF_F_RXCSUM))
2794 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2795 OCTNET_CMD_RXCSUM_DISABLE);
2797 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2798 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2799 !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2800 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2801 OCTNET_CMD_VLAN_FILTER_ENABLE);
2802 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2803 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2804 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2805 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2806 OCTNET_CMD_VLAN_FILTER_DISABLE);
2811 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2812 u8 *mac, bool is_admin_assigned)
2814 struct lio *lio = GET_LIO(netdev);
2815 struct octeon_device *oct = lio->oct_dev;
2816 struct octnic_ctrl_pkt nctrl;
2819 if (!is_valid_ether_addr(mac))
2822 if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2825 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2828 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2829 /* vfidx is 0 based, but vf_num (param1) is 1 based */
2830 nctrl.ncmd.s.param1 = vfidx + 1;
2831 nctrl.ncmd.s.more = 1;
2832 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2833 nctrl.netpndev = (u64)netdev;
2834 if (is_admin_assigned) {
2835 nctrl.ncmd.s.param2 = true;
2836 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2840 /* The MAC Address is presented in network byte order. */
2841 ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2843 oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2845 ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2852 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2854 struct lio *lio = GET_LIO(netdev);
2855 struct octeon_device *oct = lio->oct_dev;
2858 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2861 retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2863 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2868 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2871 struct lio *lio = GET_LIO(netdev);
2872 struct octeon_device *oct = lio->oct_dev;
2873 struct octnic_ctrl_pkt nctrl;
2876 if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2877 netif_info(lio, drv, lio->netdev,
2878 "firmware does not support spoofchk\n");
2882 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2883 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2888 if (oct->sriov_info.vf_spoofchk[vfidx])
2892 if (!oct->sriov_info.vf_spoofchk[vfidx])
2896 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2897 nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2898 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2899 nctrl.ncmd.s.param1 =
2900 vfidx + 1; /* vfidx is 0 based,
2901 * but vf_num (param1) is 1 based
2903 nctrl.ncmd.s.param2 = enable;
2904 nctrl.ncmd.s.more = 0;
2905 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2908 retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2911 netif_info(lio, drv, lio->netdev,
2912 "Failed to set VF %d spoofchk %s\n", vfidx,
2913 enable ? "on" : "off");
2917 oct->sriov_info.vf_spoofchk[vfidx] = enable;
2918 netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2919 enable ? "on" : "off");
2924 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2925 u16 vlan, u8 qos, __be16 vlan_proto)
2927 struct lio *lio = GET_LIO(netdev);
2928 struct octeon_device *oct = lio->oct_dev;
2929 struct octnic_ctrl_pkt nctrl;
2933 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2936 if (vlan_proto != htons(ETH_P_8021Q))
2937 return -EPROTONOSUPPORT;
2939 if (vlan >= VLAN_N_VID || qos > 7)
2943 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2947 if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2950 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2953 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2955 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2957 nctrl.ncmd.s.param1 = vlantci;
2958 nctrl.ncmd.s.param2 =
2959 vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2960 nctrl.ncmd.s.more = 0;
2961 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2964 ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2971 oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2976 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2977 struct ifla_vf_info *ivi)
2979 struct lio *lio = GET_LIO(netdev);
2980 struct octeon_device *oct = lio->oct_dev;
2983 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2986 memset(ivi, 0, sizeof(struct ifla_vf_info));
2989 macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2990 ether_addr_copy(&ivi->mac[0], macaddr);
2991 ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
2992 ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
2993 if (oct->sriov_info.trusted_vf.active &&
2994 oct->sriov_info.trusted_vf.id == vfidx)
2995 ivi->trusted = true;
2997 ivi->trusted = false;
2998 ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
2999 ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
3000 ivi->max_tx_rate = lio->linfo.link.s.speed;
3001 ivi->min_tx_rate = 0;
3006 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
3008 struct octeon_device *oct = lio->oct_dev;
3009 struct octeon_soft_command *sc;
3012 sc = octeon_alloc_soft_command(oct, 0, 16, 0);
3016 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
3018 /* vfidx is 0 based, but vf_num (param1) is 1 based */
3019 octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
3020 OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
3023 init_completion(&sc->complete);
3024 sc->sc_status = OCTEON_REQUEST_PENDING;
3026 retval = octeon_send_soft_command(oct, sc);
3027 if (retval == IQ_SEND_FAILED) {
3028 octeon_free_soft_command(oct, sc);
3031 /* Wait for response or timeout */
3032 retval = wait_for_sc_completion_timeout(oct, sc, 0);
3036 WRITE_ONCE(sc->caller_is_done, true);
3042 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3045 struct lio *lio = GET_LIO(netdev);
3046 struct octeon_device *oct = lio->oct_dev;
3048 if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3049 /* trusted vf is not supported by firmware older than 1.7.1 */
3053 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3054 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3061 if (oct->sriov_info.trusted_vf.active &&
3062 oct->sriov_info.trusted_vf.id == vfidx)
3065 if (oct->sriov_info.trusted_vf.active) {
3066 netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3072 if (!oct->sriov_info.trusted_vf.active)
3076 if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3078 oct->sriov_info.trusted_vf.id = vfidx;
3079 oct->sriov_info.trusted_vf.active = true;
3081 oct->sriov_info.trusted_vf.active = false;
3084 netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3085 setting ? "" : "not ");
3087 netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3094 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3097 struct lio *lio = GET_LIO(netdev);
3098 struct octeon_device *oct = lio->oct_dev;
3099 struct octnic_ctrl_pkt nctrl;
3102 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3105 if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3108 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3109 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3110 nctrl.ncmd.s.param1 =
3111 vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3112 nctrl.ncmd.s.param2 = linkstate;
3113 nctrl.ncmd.s.more = 0;
3114 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3117 ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3120 oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3128 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3130 struct lio_devlink_priv *priv;
3131 struct octeon_device *oct;
3133 priv = devlink_priv(devlink);
3136 *mode = oct->eswitch_mode;
3142 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3143 struct netlink_ext_ack *extack)
3145 struct lio_devlink_priv *priv;
3146 struct octeon_device *oct;
3149 priv = devlink_priv(devlink);
3152 if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3155 if (oct->eswitch_mode == mode)
3159 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3160 oct->eswitch_mode = mode;
3161 ret = lio_vf_rep_create(oct);
3164 case DEVLINK_ESWITCH_MODE_LEGACY:
3165 lio_vf_rep_destroy(oct);
3166 oct->eswitch_mode = mode;
3176 static const struct devlink_ops liquidio_devlink_ops = {
3177 .eswitch_mode_get = liquidio_eswitch_mode_get,
3178 .eswitch_mode_set = liquidio_eswitch_mode_set,
3182 liquidio_get_port_parent_id(struct net_device *dev,
3183 struct netdev_phys_item_id *ppid)
3185 struct lio *lio = GET_LIO(dev);
3186 struct octeon_device *oct = lio->oct_dev;
3188 if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3191 ppid->id_len = ETH_ALEN;
3192 ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3197 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3198 struct ifla_vf_stats *vf_stats)
3200 struct lio *lio = GET_LIO(netdev);
3201 struct octeon_device *oct = lio->oct_dev;
3202 struct oct_vf_stats stats;
3205 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3208 memset(&stats, 0, sizeof(struct oct_vf_stats));
3209 ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3211 vf_stats->rx_packets = stats.rx_packets;
3212 vf_stats->tx_packets = stats.tx_packets;
3213 vf_stats->rx_bytes = stats.rx_bytes;
3214 vf_stats->tx_bytes = stats.tx_bytes;
3215 vf_stats->broadcast = stats.broadcast;
3216 vf_stats->multicast = stats.multicast;
3222 static const struct net_device_ops lionetdevops = {
3223 .ndo_open = liquidio_open,
3224 .ndo_stop = liquidio_stop,
3225 .ndo_start_xmit = liquidio_xmit,
3226 .ndo_get_stats64 = liquidio_get_stats64,
3227 .ndo_set_mac_address = liquidio_set_mac,
3228 .ndo_set_rx_mode = liquidio_set_mcast_list,
3229 .ndo_tx_timeout = liquidio_tx_timeout,
3231 .ndo_vlan_rx_add_vid = liquidio_vlan_rx_add_vid,
3232 .ndo_vlan_rx_kill_vid = liquidio_vlan_rx_kill_vid,
3233 .ndo_change_mtu = liquidio_change_mtu,
3234 .ndo_eth_ioctl = liquidio_ioctl,
3235 .ndo_fix_features = liquidio_fix_features,
3236 .ndo_set_features = liquidio_set_features,
3237 .ndo_set_vf_mac = liquidio_set_vf_mac,
3238 .ndo_set_vf_vlan = liquidio_set_vf_vlan,
3239 .ndo_get_vf_config = liquidio_get_vf_config,
3240 .ndo_set_vf_spoofchk = liquidio_set_vf_spoofchk,
3241 .ndo_set_vf_trust = liquidio_set_vf_trust,
3242 .ndo_set_vf_link_state = liquidio_set_vf_link_state,
3243 .ndo_get_vf_stats = liquidio_get_vf_stats,
3244 .ndo_get_port_parent_id = liquidio_get_port_parent_id,
3248 * liquidio_init - Entry point for the liquidio module
3250 static int __init liquidio_init(void)
3253 struct handshake *hs;
3255 init_completion(&first_stage);
3257 octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3259 if (liquidio_init_pci())
3262 wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3264 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3267 wait_for_completion(&hs->init);
3269 /* init handshake failed */
3270 dev_err(&hs->pci_dev->dev,
3271 "Failed to init device\n");
3272 liquidio_deinit_pci();
3278 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3281 wait_for_completion_timeout(&hs->started,
3282 msecs_to_jiffies(30000));
3283 if (!hs->started_ok) {
3284 /* starter handshake failed */
3285 dev_err(&hs->pci_dev->dev,
3286 "Firmware failed to start\n");
3287 liquidio_deinit_pci();
3296 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3298 struct octeon_device *oct = (struct octeon_device *)buf;
3299 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3301 union oct_link_status *ls;
3304 if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3305 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3306 recv_pkt->buffer_size[0],
3307 recv_pkt->rh.r_nic_info.gmxport);
3311 gmxport = recv_pkt->rh.r_nic_info.gmxport;
3312 ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3313 OCT_DROQ_INFO_SIZE);
3315 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3316 for (i = 0; i < oct->ifcount; i++) {
3317 if (oct->props[i].gmxport == gmxport) {
3318 update_link_status(oct->props[i].netdev, ls);
3324 for (i = 0; i < recv_pkt->buffer_count; i++)
3325 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3326 octeon_free_recv_info(recv_info);
3331 * setup_nic_devices - Setup network interfaces
3332 * @octeon_dev: octeon device
3334 * Called during init time for each device. It assumes the NIC
3335 * is already up and running. The link information for each
3336 * interface is passed in link_info.
3338 static int setup_nic_devices(struct octeon_device *octeon_dev)
3340 struct lio *lio = NULL;
3341 struct net_device *netdev;
3342 u8 mac[6], i, j, *fw_ver, *micro_ver;
3343 unsigned long micro;
3345 struct octeon_soft_command *sc;
3346 struct liquidio_if_cfg_resp *resp;
3347 struct octdev_props *props;
3348 int retval, num_iqueues, num_oqueues;
3349 int max_num_queues = 0;
3350 union oct_nic_if_cfg if_cfg;
3351 unsigned int base_queue;
3352 unsigned int gmx_port_id;
3353 u32 resp_size, data_size;
3355 struct lio_version *vdata;
3356 struct devlink *devlink;
3357 struct lio_devlink_priv *lio_devlink;
3359 /* This is to handle link status changes */
3360 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3362 lio_nic_info, octeon_dev);
3364 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3365 * They are handled directly.
3367 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3370 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3373 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3374 free_netsgbuf_with_resp);
3376 for (i = 0; i < octeon_dev->ifcount; i++) {
3377 resp_size = sizeof(struct liquidio_if_cfg_resp);
3378 data_size = sizeof(struct lio_version);
3379 sc = (struct octeon_soft_command *)
3380 octeon_alloc_soft_command(octeon_dev, data_size,
3382 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3383 vdata = (struct lio_version *)sc->virtdptr;
3385 *((u64 *)vdata) = 0;
3386 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3387 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3388 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3390 if (OCTEON_CN23XX_PF(octeon_dev)) {
3391 num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3392 num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3393 base_queue = octeon_dev->sriov_info.pf_srn;
3395 gmx_port_id = octeon_dev->pf_num;
3396 ifidx_or_pfnum = octeon_dev->pf_num;
3398 num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3399 octeon_get_conf(octeon_dev), i);
3400 num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3401 octeon_get_conf(octeon_dev), i);
3402 base_queue = CFG_GET_BASE_QUE_NIC_IF(
3403 octeon_get_conf(octeon_dev), i);
3404 gmx_port_id = CFG_GET_GMXID_NIC_IF(
3405 octeon_get_conf(octeon_dev), i);
3409 dev_dbg(&octeon_dev->pci_dev->dev,
3410 "requesting config for interface %d, iqs %d, oqs %d\n",
3411 ifidx_or_pfnum, num_iqueues, num_oqueues);
3414 if_cfg.s.num_iqueues = num_iqueues;
3415 if_cfg.s.num_oqueues = num_oqueues;
3416 if_cfg.s.base_queue = base_queue;
3417 if_cfg.s.gmx_port_id = gmx_port_id;
3421 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3422 OPCODE_NIC_IF_CFG, 0,
3425 init_completion(&sc->complete);
3426 sc->sc_status = OCTEON_REQUEST_PENDING;
3428 retval = octeon_send_soft_command(octeon_dev, sc);
3429 if (retval == IQ_SEND_FAILED) {
3430 dev_err(&octeon_dev->pci_dev->dev,
3431 "iq/oq config failed status: %x\n",
3433 /* Soft instr is freed by driver in case of failure. */
3434 octeon_free_soft_command(octeon_dev, sc);
3438 /* Sleep on a wait queue till the cond flag indicates that the
3439 * response arrived or timed-out.
3441 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3445 retval = resp->status;
3447 dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3448 WRITE_ONCE(sc->caller_is_done, true);
3449 goto setup_nic_dev_done;
3451 snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3453 resp->cfg_info.liquidio_firmware_version);
3455 /* Verify f/w version (in case of 'auto' loading from flash) */
3456 fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3457 if (memcmp(LIQUIDIO_BASE_VERSION,
3459 strlen(LIQUIDIO_BASE_VERSION))) {
3460 dev_err(&octeon_dev->pci_dev->dev,
3461 "Unmatched firmware version. Expected %s.x, got %s.\n",
3462 LIQUIDIO_BASE_VERSION, fw_ver);
3463 WRITE_ONCE(sc->caller_is_done, true);
3464 goto setup_nic_dev_done;
3465 } else if (atomic_read(octeon_dev->adapter_fw_state) ==
3467 dev_info(&octeon_dev->pci_dev->dev,
3468 "Using auto-loaded firmware version %s.\n",
3472 /* extract micro version field; point past '<maj>.<min>.' */
3473 micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3474 if (kstrtoul(micro_ver, 10, µ) != 0)
3476 octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3477 octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3478 octeon_dev->fw_info.ver.rev = micro;
3480 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3481 (sizeof(struct liquidio_if_cfg_info)) >> 3);
3483 num_iqueues = hweight64(resp->cfg_info.iqmask);
3484 num_oqueues = hweight64(resp->cfg_info.oqmask);
3486 if (!(num_iqueues) || !(num_oqueues)) {
3487 dev_err(&octeon_dev->pci_dev->dev,
3488 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3489 resp->cfg_info.iqmask,
3490 resp->cfg_info.oqmask);
3491 WRITE_ONCE(sc->caller_is_done, true);
3492 goto setup_nic_dev_done;
3495 if (OCTEON_CN6XXX(octeon_dev)) {
3496 max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3498 } else if (OCTEON_CN23XX_PF(octeon_dev)) {
3499 max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3503 dev_dbg(&octeon_dev->pci_dev->dev,
3504 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3505 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3506 num_iqueues, num_oqueues, max_num_queues);
3507 netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3510 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3511 WRITE_ONCE(sc->caller_is_done, true);
3512 goto setup_nic_dev_done;
3515 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3517 /* Associate the routines that will handle different
3520 netdev->netdev_ops = &lionetdevops;
3522 retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3524 dev_err(&octeon_dev->pci_dev->dev,
3525 "setting real number rx failed\n");
3526 WRITE_ONCE(sc->caller_is_done, true);
3527 goto setup_nic_dev_free;
3530 retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3532 dev_err(&octeon_dev->pci_dev->dev,
3533 "setting real number tx failed\n");
3534 WRITE_ONCE(sc->caller_is_done, true);
3535 goto setup_nic_dev_free;
3538 lio = GET_LIO(netdev);
3540 memset(lio, 0, sizeof(struct lio));
3542 lio->ifidx = ifidx_or_pfnum;
3544 props = &octeon_dev->props[i];
3545 props->gmxport = resp->cfg_info.linfo.gmxport;
3546 props->netdev = netdev;
3548 lio->linfo.num_rxpciq = num_oqueues;
3549 lio->linfo.num_txpciq = num_iqueues;
3550 for (j = 0; j < num_oqueues; j++) {
3551 lio->linfo.rxpciq[j].u64 =
3552 resp->cfg_info.linfo.rxpciq[j].u64;
3554 for (j = 0; j < num_iqueues; j++) {
3555 lio->linfo.txpciq[j].u64 =
3556 resp->cfg_info.linfo.txpciq[j].u64;
3558 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3559 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3560 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3562 WRITE_ONCE(sc->caller_is_done, true);
3564 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3566 if (OCTEON_CN23XX_PF(octeon_dev) ||
3567 OCTEON_CN6XXX(octeon_dev)) {
3568 lio->dev_capability = NETIF_F_HIGHDMA
3571 | NETIF_F_SG | NETIF_F_RXCSUM
3573 | NETIF_F_TSO | NETIF_F_TSO6
3576 netif_set_tso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3578 /* Copy of transmit encapsulation capabilities:
3579 * TSO, TSO6, Checksums for this device
3581 lio->enc_dev_capability = NETIF_F_IP_CSUM
3583 | NETIF_F_GSO_UDP_TUNNEL
3584 | NETIF_F_HW_CSUM | NETIF_F_SG
3586 | NETIF_F_TSO | NETIF_F_TSO6
3589 netdev->hw_enc_features = (lio->enc_dev_capability &
3592 netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels;
3594 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3596 netdev->vlan_features = lio->dev_capability;
3597 /* Add any unchangeable hw features */
3598 lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER |
3599 NETIF_F_HW_VLAN_CTAG_RX |
3600 NETIF_F_HW_VLAN_CTAG_TX;
3602 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3604 netdev->hw_features = lio->dev_capability;
3605 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3606 netdev->hw_features = netdev->hw_features &
3607 ~NETIF_F_HW_VLAN_CTAG_RX;
3609 /* MTU range: 68 - 16000 */
3610 netdev->min_mtu = LIO_MIN_MTU_SIZE;
3611 netdev->max_mtu = LIO_MAX_MTU_SIZE;
3613 /* Point to the properties for octeon device to which this
3614 * interface belongs.
3616 lio->oct_dev = octeon_dev;
3617 lio->octprops = props;
3618 lio->netdev = netdev;
3620 dev_dbg(&octeon_dev->pci_dev->dev,
3621 "if%d gmx: %d hw_addr: 0x%llx\n", i,
3622 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3624 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3627 eth_random_addr(vfmac);
3628 if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3629 dev_err(&octeon_dev->pci_dev->dev,
3630 "Error setting VF%d MAC address\n",
3632 goto setup_nic_dev_free;
3636 /* 64-bit swap required on LE machines */
3637 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3638 for (j = 0; j < 6; j++)
3639 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3641 /* Copy MAC Address to OS network device structure */
3643 eth_hw_addr_set(netdev, mac);
3645 /* By default all interfaces on a single Octeon uses the same
3648 lio->txq = lio->linfo.txpciq[0].s.q_no;
3649 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3650 if (liquidio_setup_io_queues(octeon_dev, i,
3651 lio->linfo.num_txpciq,
3652 lio->linfo.num_rxpciq)) {
3653 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3654 goto setup_nic_dev_free;
3657 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3659 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3660 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3662 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3663 dev_err(&octeon_dev->pci_dev->dev,
3664 "Gather list allocation failed\n");
3665 goto setup_nic_dev_free;
3668 /* Register ethtool support */
3669 liquidio_set_ethtool_ops(netdev);
3670 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3671 octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3673 octeon_dev->priv_flags = 0x0;
3675 if (netdev->features & NETIF_F_LRO)
3676 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3677 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3679 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3680 OCTNET_CMD_VLAN_FILTER_ENABLE);
3682 if ((debug != -1) && (debug & NETIF_MSG_HW))
3683 liquidio_set_feature(netdev,
3684 OCTNET_CMD_VERBOSE_ENABLE, 0);
3686 if (setup_link_status_change_wq(netdev))
3687 goto setup_nic_dev_free;
3689 if ((octeon_dev->fw_info.app_cap_flags &
3690 LIQUIDIO_TIME_SYNC_CAP) &&
3691 setup_sync_octeon_time_wq(netdev))
3692 goto setup_nic_dev_free;
3694 if (setup_rx_oom_poll_fn(netdev))
3695 goto setup_nic_dev_free;
3697 /* Register the network device with the OS */
3698 if (register_netdev(netdev)) {
3699 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3700 goto setup_nic_dev_free;
3703 dev_dbg(&octeon_dev->pci_dev->dev,
3704 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3705 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3706 netif_carrier_off(netdev);
3707 lio->link_changes++;
3709 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3711 /* Sending command to firmware to enable Rx checksum offload
3712 * by default at the time of setup of Liquidio driver for
3715 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3716 OCTNET_CMD_RXCSUM_ENABLE);
3717 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3718 OCTNET_CMD_TXCSUM_ENABLE);
3720 dev_dbg(&octeon_dev->pci_dev->dev,
3721 "NIC ifidx:%d Setup successful\n", i);
3723 if (octeon_dev->subsystem_id ==
3724 OCTEON_CN2350_25GB_SUBSYS_ID ||
3725 octeon_dev->subsystem_id ==
3726 OCTEON_CN2360_25GB_SUBSYS_ID) {
3727 cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3728 octeon_dev->fw_info.ver.min,
3729 octeon_dev->fw_info.ver.rev);
3731 /* speed control unsupported in f/w older than 1.7.2 */
3732 if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3733 dev_info(&octeon_dev->pci_dev->dev,
3734 "speed setting not supported by f/w.");
3735 octeon_dev->speed_setting = 25;
3736 octeon_dev->no_speed_setting = 1;
3738 liquidio_get_speed(lio);
3741 if (octeon_dev->speed_setting == 0) {
3742 octeon_dev->speed_setting = 25;
3743 octeon_dev->no_speed_setting = 1;
3746 octeon_dev->no_speed_setting = 1;
3747 octeon_dev->speed_setting = 10;
3749 octeon_dev->speed_boot = octeon_dev->speed_setting;
3751 /* don't read FEC setting if unsupported by f/w (see above) */
3752 if (octeon_dev->speed_boot == 25 &&
3753 !octeon_dev->no_speed_setting) {
3754 liquidio_get_fec(lio);
3755 octeon_dev->props[lio->ifidx].fec_boot =
3756 octeon_dev->props[lio->ifidx].fec;
3760 device_lock(&octeon_dev->pci_dev->dev);
3761 devlink = devlink_alloc(&liquidio_devlink_ops,
3762 sizeof(struct lio_devlink_priv),
3763 &octeon_dev->pci_dev->dev);
3765 device_unlock(&octeon_dev->pci_dev->dev);
3766 dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3767 goto setup_nic_dev_free;
3770 lio_devlink = devlink_priv(devlink);
3771 lio_devlink->oct = octeon_dev;
3773 octeon_dev->devlink = devlink;
3774 octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3775 devlink_register(devlink);
3776 device_unlock(&octeon_dev->pci_dev->dev);
3783 dev_err(&octeon_dev->pci_dev->dev,
3784 "NIC ifidx:%d Setup failed\n", i);
3785 liquidio_destroy_nic_device(octeon_dev, i);
3793 #ifdef CONFIG_PCI_IOV
3794 static int octeon_enable_sriov(struct octeon_device *oct)
3796 unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3797 struct pci_dev *vfdev;
3801 if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3802 err = pci_enable_sriov(oct->pci_dev,
3803 oct->sriov_info.num_vfs_alloced);
3805 dev_err(&oct->pci_dev->dev,
3806 "OCTEON: Failed to enable PCI sriov: %d\n",
3808 oct->sriov_info.num_vfs_alloced = 0;
3811 oct->sriov_info.sriov_enabled = 1;
3813 /* init lookup table that maps DPI ring number to VF pci_dev
3817 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3818 OCTEON_CN23XX_VF_VID, NULL);
3820 if (vfdev->is_virtfn &&
3821 (vfdev->physfn == oct->pci_dev)) {
3822 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3824 u += oct->sriov_info.rings_per_vf;
3826 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3827 OCTEON_CN23XX_VF_VID, vfdev);
3831 return num_vfs_alloced;
3834 static int lio_pci_sriov_disable(struct octeon_device *oct)
3838 if (pci_vfs_assigned(oct->pci_dev)) {
3839 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3843 pci_disable_sriov(oct->pci_dev);
3846 while (u < MAX_POSSIBLE_VFS) {
3847 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3848 u += oct->sriov_info.rings_per_vf;
3851 oct->sriov_info.num_vfs_alloced = 0;
3852 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3858 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3860 struct octeon_device *oct = pci_get_drvdata(dev);
3863 if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3864 (oct->sriov_info.sriov_enabled)) {
3865 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3866 oct->pf_num, num_vfs);
3871 lio_vf_rep_destroy(oct);
3872 ret = lio_pci_sriov_disable(oct);
3873 } else if (num_vfs > oct->sriov_info.max_vfs) {
3874 dev_err(&oct->pci_dev->dev,
3875 "OCTEON: Max allowed VFs:%d user requested:%d",
3876 oct->sriov_info.max_vfs, num_vfs);
3879 oct->sriov_info.num_vfs_alloced = num_vfs;
3880 ret = octeon_enable_sriov(oct);
3881 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3882 oct->pf_num, num_vfs);
3883 ret = lio_vf_rep_create(oct);
3885 dev_info(&oct->pci_dev->dev,
3886 "vf representor create failed");
3894 * liquidio_init_nic_module - initialize the NIC
3895 * @oct: octeon device
3897 * This initialization routine is called once the Octeon device application is
3900 static int liquidio_init_nic_module(struct octeon_device *oct)
3903 int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3905 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3907 /* only default iq and oq were initialized
3908 * initialize the rest as well
3910 /* run port_config command for each port */
3911 oct->ifcount = num_nic_ports;
3913 memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3915 for (i = 0; i < MAX_OCTEON_LINKS; i++)
3916 oct->props[i].gmxport = -1;
3918 retval = setup_nic_devices(oct);
3920 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3921 goto octnet_init_failure;
3924 /* Call vf_rep_modinit if the firmware is switchdev capable
3925 * and do it from the first liquidio function probed.
3927 if (!oct->octeon_id &&
3928 oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3929 retval = lio_vf_rep_modinit();
3931 liquidio_stop_nic_module(oct);
3932 goto octnet_init_failure;
3936 liquidio_ptp_init(oct);
3938 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3942 octnet_init_failure:
3950 * nic_starter - finish init
3951 * @work: work struct work_struct
3953 * starter callback that invokes the remaining initialization work after the NIC is up and running.
3955 static void nic_starter(struct work_struct *work)
3957 struct octeon_device *oct;
3958 struct cavium_wk *wk = (struct cavium_wk *)work;
3960 oct = (struct octeon_device *)wk->ctxptr;
3962 if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3965 /* If the status of the device is CORE_OK, the core
3966 * application has reported its application type. Call
3967 * any registered handlers now and move to the RUNNING
3970 if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3971 schedule_delayed_work(&oct->nic_poll_work.work,
3972 LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3976 atomic_set(&oct->status, OCT_DEV_RUNNING);
3978 if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3979 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3981 if (liquidio_init_nic_module(oct))
3982 dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3984 handshake[oct->octeon_id].started_ok = 1;
3986 dev_err(&oct->pci_dev->dev,
3987 "Unexpected application running on NIC (%d). Check firmware.\n",
3991 complete(&handshake[oct->octeon_id].started);
3995 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
3997 struct octeon_device *oct = (struct octeon_device *)buf;
3998 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3999 int i, notice, vf_idx;
4003 notice = recv_pkt->rh.r.ossp;
4004 data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4006 /* the first 64-bit word of data is the vf_num */
4008 octeon_swap_8B_data(&vf_num, 1);
4009 vf_idx = (int)vf_num - 1;
4011 cores_crashed = READ_ONCE(oct->cores_crashed);
4013 if (notice == VF_DRV_LOADED) {
4014 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4015 oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4016 dev_info(&oct->pci_dev->dev,
4017 "driver for VF%d was loaded\n", vf_idx);
4019 try_module_get(THIS_MODULE);
4021 } else if (notice == VF_DRV_REMOVED) {
4022 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4023 oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4024 dev_info(&oct->pci_dev->dev,
4025 "driver for VF%d was removed\n", vf_idx);
4027 module_put(THIS_MODULE);
4029 } else if (notice == VF_DRV_MACADDR_CHANGED) {
4030 u8 *b = (u8 *)&data[1];
4032 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4033 dev_info(&oct->pci_dev->dev,
4034 "VF driver changed VF%d's MAC address to %pM\n",
4038 for (i = 0; i < recv_pkt->buffer_count; i++)
4039 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4040 octeon_free_recv_info(recv_info);
4046 * octeon_device_init - Device initialization for each Octeon device that is probed
4047 * @octeon_dev: octeon device
4049 static int octeon_device_init(struct octeon_device *octeon_dev)
4052 char bootcmd[] = "\n";
4053 char *dbg_enb = NULL;
4054 enum lio_fw_state fw_state;
4055 struct octeon_device_priv *oct_priv = octeon_dev->priv;
4056 atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4058 /* Enable access to the octeon device and make its DMA capability
4061 if (octeon_pci_os_setup(octeon_dev))
4064 atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4066 /* Identify the Octeon type and map the BAR address space. */
4067 if (octeon_chip_specific_setup(octeon_dev)) {
4068 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4072 atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4074 /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4075 * since that is what is required for the reference to be removed
4076 * during de-initialization (see 'octeon_destroy_resources').
4078 octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4079 PCI_SLOT(octeon_dev->pci_dev->devfn),
4080 PCI_FUNC(octeon_dev->pci_dev->devfn),
4083 octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4085 /* CN23XX supports preloaded firmware if the following is true:
4087 * The adapter indicates that firmware is currently running AND
4088 * 'fw_type' is 'auto'.
4090 * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4092 if (OCTEON_CN23XX_PF(octeon_dev) &&
4093 cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4094 atomic_cmpxchg(octeon_dev->adapter_fw_state,
4095 FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4098 /* If loading firmware, only first device of adapter needs to do so. */
4099 fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4100 FW_NEEDS_TO_BE_LOADED,
4101 FW_IS_BEING_LOADED);
4103 /* Here, [local variable] 'fw_state' is set to one of:
4105 * FW_IS_PRELOADED: No firmware is to be loaded (see above)
4106 * FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4107 * firmware to the adapter.
4108 * FW_IS_BEING_LOADED: The driver's second instance will not load
4109 * firmware to the adapter.
4112 /* Prior to f/w load, perform a soft reset of the Octeon device;
4113 * if error resetting, return w/error.
4115 if (fw_state == FW_NEEDS_TO_BE_LOADED)
4116 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4119 /* Initialize the dispatch mechanism used to push packets arriving on
4120 * Octeon Output queues.
4122 if (octeon_init_dispatch_list(octeon_dev))
4125 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4126 OPCODE_NIC_CORE_DRV_ACTIVE,
4127 octeon_core_drv_init,
4130 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4131 OPCODE_NIC_VF_DRV_NOTICE,
4132 octeon_recv_vf_drv_notice, octeon_dev);
4133 INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4134 octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4135 schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4136 LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4138 atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4140 if (octeon_set_io_queues_off(octeon_dev)) {
4141 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4145 if (OCTEON_CN23XX_PF(octeon_dev)) {
4146 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4148 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4153 /* Initialize soft command buffer pool
4155 if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4156 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4159 atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4161 /* Setup the data structures that manage this Octeon's Input queues. */
4162 if (octeon_setup_instr_queues(octeon_dev)) {
4163 dev_err(&octeon_dev->pci_dev->dev,
4164 "instruction queue initialization failed\n");
4167 atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4169 /* Initialize lists to manage the requests of different types that
4170 * arrive from user & kernel applications for this octeon device.
4172 if (octeon_setup_response_list(octeon_dev)) {
4173 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4176 atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4178 if (octeon_setup_output_queues(octeon_dev)) {
4179 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4183 atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4185 if (OCTEON_CN23XX_PF(octeon_dev)) {
4186 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4187 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4190 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4192 if (octeon_allocate_ioq_vector
4194 octeon_dev->sriov_info.num_pf_rings)) {
4195 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4198 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4201 /* The input and output queue registers were setup earlier (the
4202 * queues were not enabled). Any additional registers
4203 * that need to be programmed should be done now.
4205 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4207 dev_err(&octeon_dev->pci_dev->dev,
4208 "Failed to configure device registers\n");
4213 /* Initialize the tasklet that handles output queue packet processing.*/
4214 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4215 tasklet_setup(&oct_priv->droq_tasklet, octeon_droq_bh);
4217 /* Setup the interrupt handler and record the INT SUM register address
4219 if (octeon_setup_interrupt(octeon_dev,
4220 octeon_dev->sriov_info.num_pf_rings))
4223 /* Enable Octeon device interrupts */
4224 octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4226 atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4228 /* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4229 * the output queue is enabled.
4230 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4231 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4232 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4233 * before any credits have been issued, causing the ring to be reset
4234 * (and the f/w appear to never have started).
4236 for (j = 0; j < octeon_dev->num_oqs; j++)
4237 writel(octeon_dev->droq[j]->max_count,
4238 octeon_dev->droq[j]->pkts_credit_reg);
4240 /* Enable the input and output queues for this Octeon device */
4241 ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4243 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4247 atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4249 if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4250 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4252 dev_info(&octeon_dev->pci_dev->dev,
4253 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4256 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4258 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4259 while (!ddr_timeout) {
4260 set_current_state(TASK_INTERRUPTIBLE);
4261 if (schedule_timeout(HZ / 10)) {
4262 /* user probably pressed Control-C */
4266 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4268 dev_err(&octeon_dev->pci_dev->dev,
4269 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4274 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4275 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4279 /* Divert uboot to take commands from host instead. */
4280 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4282 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4283 ret = octeon_init_consoles(octeon_dev);
4285 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4288 /* If console debug enabled, specify empty string to use default
4289 * enablement ELSE specify NULL string for 'disabled'.
4291 dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4292 ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4294 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4296 } else if (octeon_console_debug_enabled(0)) {
4297 /* If console was added AND we're logging console output
4298 * then set our console print function.
4300 octeon_dev->console[0].print = octeon_dbg_console_print;
4303 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4305 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4306 ret = load_firmware(octeon_dev);
4308 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4312 atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4315 handshake[octeon_dev->octeon_id].init_ok = 1;
4316 complete(&handshake[octeon_dev->octeon_id].init);
4318 atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4319 oct_priv->dev = octeon_dev;
4325 * octeon_dbg_console_print - Debug console print function
4326 * @oct: octeon device
4327 * @console_num: console number
4328 * @prefix: first portion of line to display
4329 * @suffix: second portion of line to display
4331 * The OCTEON debug console outputs entire lines (excluding '\n').
4332 * Normally, the line will be passed in the 'prefix' parameter.
4333 * However, due to buffering, it is possible for a line to be split into two
4334 * parts, in which case they will be passed as the 'prefix' parameter and
4335 * 'suffix' parameter.
4337 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4338 char *prefix, char *suffix)
4340 if (prefix && suffix)
4341 dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4344 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4346 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4352 * liquidio_exit - Exits the module
4354 static void __exit liquidio_exit(void)
4356 liquidio_deinit_pci();
4358 pr_info("LiquidIO network module is now unloaded\n");
4361 module_init(liquidio_init);
4362 module_exit(liquidio_exit);