1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
4 #include <linux/etherdevice.h>
5 #include <linux/of_net.h>
8 #include <generated/utsrelease.h>
12 #include "i40e_diag.h"
14 #include <net/udp_tunnel.h>
15 #include <net/xdp_sock_drv.h>
16 /* All i40e tracepoints are defined by the include below, which
17 * must be included exactly once across the whole kernel with
18 * CREATE_TRACE_POINTS defined
20 #define CREATE_TRACE_POINTS
21 #include "i40e_trace.h"
23 const char i40e_driver_name[] = "i40e";
24 static const char i40e_driver_string[] =
25 "Intel(R) Ethernet Connection XL710 Network Driver";
27 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
29 /* a bit of forward declarations */
30 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
31 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
32 static int i40e_add_vsi(struct i40e_vsi *vsi);
33 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
34 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
35 static int i40e_setup_misc_vector(struct i40e_pf *pf);
36 static void i40e_determine_queue_usage(struct i40e_pf *pf);
37 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
38 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
39 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
41 static int i40e_reset(struct i40e_pf *pf);
42 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
43 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
44 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
45 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
46 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
47 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
48 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
49 static int i40e_get_capabilities(struct i40e_pf *pf,
50 enum i40e_admin_queue_opc list_type);
51 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
53 /* i40e_pci_tbl - PCI Device ID Table
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
58 * Class, Class Mask, private data (not used) }
60 static const struct pci_device_id i40e_pci_tbl[] = {
61 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
85 /* required last entry */
88 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
90 #define I40E_MAX_VF_COUNT 128
91 static int debug = -1;
92 module_param(debug, uint, 0);
93 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
95 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
96 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
97 MODULE_LICENSE("GPL v2");
99 static struct workqueue_struct *i40e_wq;
101 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
102 struct net_device *netdev, int delta)
104 struct netdev_hw_addr_list *ha_list;
105 struct netdev_hw_addr *ha;
110 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
111 ha_list = &netdev->uc;
113 ha_list = &netdev->mc;
115 netdev_hw_addr_list_for_each(ha, ha_list) {
116 if (ether_addr_equal(ha->addr, f->macaddr)) {
117 ha->refcount += delta;
118 if (ha->refcount <= 0)
126 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
127 * @hw: pointer to the HW structure
128 * @mem: ptr to mem struct to fill out
129 * @size: size of memory requested
130 * @alignment: what to align the allocation to
132 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
133 u64 size, u32 alignment)
135 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
137 mem->size = ALIGN(size, alignment);
138 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
147 * i40e_free_dma_mem_d - OS specific memory free for shared code
148 * @hw: pointer to the HW structure
149 * @mem: ptr to mem struct to free
151 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
153 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
155 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
164 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
165 * @hw: pointer to the HW structure
166 * @mem: ptr to mem struct to fill out
167 * @size: size of memory requested
169 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
173 mem->va = kzalloc(size, GFP_KERNEL);
182 * i40e_free_virt_mem_d - OS specific memory free for shared code
183 * @hw: pointer to the HW structure
184 * @mem: ptr to mem struct to free
186 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
188 /* it's ok to kfree a NULL pointer */
197 * i40e_get_lump - find a lump of free generic resource
198 * @pf: board private structure
199 * @pile: the pile of resource to search
200 * @needed: the number of items needed
201 * @id: an owner id to stick on the items assigned
203 * Returns the base item index of the lump, or negative for error
205 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
211 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
212 dev_info(&pf->pdev->dev,
213 "param err: pile=%s needed=%d id=0x%04x\n",
214 pile ? "<valid>" : "<null>", needed, id);
218 /* Allocate last queue in the pile for FDIR VSI queue
219 * so it doesn't fragment the qp_pile
221 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
222 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
223 dev_err(&pf->pdev->dev,
224 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
225 pile->num_entries - 1);
228 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
229 return pile->num_entries - 1;
233 while (i < pile->num_entries) {
234 /* skip already allocated entries */
235 if (pile->list[i] & I40E_PILE_VALID_BIT) {
240 /* do we have enough in this lump? */
241 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
242 if (pile->list[i+j] & I40E_PILE_VALID_BIT)
247 /* there was enough, so assign it to the requestor */
248 for (j = 0; j < needed; j++)
249 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
254 /* not enough, so skip over it and continue looking */
262 * i40e_put_lump - return a lump of generic resource
263 * @pile: the pile of resource to search
264 * @index: the base item index
265 * @id: the owner id of the items assigned
267 * Returns the count of items in the lump
269 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
271 int valid_id = (id | I40E_PILE_VALID_BIT);
275 if (!pile || index >= pile->num_entries)
279 i < pile->num_entries && pile->list[i] == valid_id;
290 * i40e_find_vsi_from_id - searches for the vsi with the given id
291 * @pf: the pf structure to search for the vsi
292 * @id: id of the vsi it is searching for
294 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
298 for (i = 0; i < pf->num_alloc_vsi; i++)
299 if (pf->vsi[i] && (pf->vsi[i]->id == id))
306 * i40e_service_event_schedule - Schedule the service task to wake up
307 * @pf: board private structure
309 * If not already scheduled, this puts the task into the work queue
311 void i40e_service_event_schedule(struct i40e_pf *pf)
313 if ((!test_bit(__I40E_DOWN, pf->state) &&
314 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
315 test_bit(__I40E_RECOVERY_MODE, pf->state))
316 queue_work(i40e_wq, &pf->service_task);
320 * i40e_tx_timeout - Respond to a Tx Hang
321 * @netdev: network interface device structure
322 * @txqueue: queue number timing out
324 * If any port has noticed a Tx timeout, it is likely that the whole
325 * device is munged, not just the one netdev port, so go for the full
328 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
330 struct i40e_netdev_priv *np = netdev_priv(netdev);
331 struct i40e_vsi *vsi = np->vsi;
332 struct i40e_pf *pf = vsi->back;
333 struct i40e_ring *tx_ring = NULL;
337 pf->tx_timeout_count++;
339 /* with txqueue index, find the tx_ring struct */
340 for (i = 0; i < vsi->num_queue_pairs; i++) {
341 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
343 vsi->tx_rings[i]->queue_index) {
344 tx_ring = vsi->tx_rings[i];
350 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
351 pf->tx_timeout_recovery_level = 1; /* reset after some time */
352 else if (time_before(jiffies,
353 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
354 return; /* don't do any new action before the next timeout */
356 /* don't kick off another recovery if one is already pending */
357 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
361 head = i40e_get_head(tx_ring);
362 /* Read interrupt register */
363 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
365 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
366 tx_ring->vsi->base_vector - 1));
368 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
370 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
371 vsi->seid, txqueue, tx_ring->next_to_clean,
372 head, tx_ring->next_to_use,
373 readl(tx_ring->tail), val);
376 pf->tx_timeout_last_recovery = jiffies;
377 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
378 pf->tx_timeout_recovery_level, txqueue);
380 switch (pf->tx_timeout_recovery_level) {
382 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
385 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
388 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
391 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
392 set_bit(__I40E_DOWN_REQUESTED, pf->state);
393 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
397 i40e_service_event_schedule(pf);
398 pf->tx_timeout_recovery_level++;
402 * i40e_get_vsi_stats_struct - Get System Network Statistics
403 * @vsi: the VSI we care about
405 * Returns the address of the device statistics structure.
406 * The statistics are actually updated from the service task.
408 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
410 return &vsi->net_stats;
414 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
415 * @ring: Tx ring to get statistics from
416 * @stats: statistics entry to be updated
418 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
419 struct rtnl_link_stats64 *stats)
425 start = u64_stats_fetch_begin_irq(&ring->syncp);
426 packets = ring->stats.packets;
427 bytes = ring->stats.bytes;
428 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
430 stats->tx_packets += packets;
431 stats->tx_bytes += bytes;
435 * i40e_get_netdev_stats_struct - Get statistics for netdev interface
436 * @netdev: network interface device structure
437 * @stats: data structure to store statistics
439 * Returns the address of the device statistics structure.
440 * The statistics are actually updated from the service task.
442 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
443 struct rtnl_link_stats64 *stats)
445 struct i40e_netdev_priv *np = netdev_priv(netdev);
446 struct i40e_vsi *vsi = np->vsi;
447 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
448 struct i40e_ring *ring;
451 if (test_bit(__I40E_VSI_DOWN, vsi->state))
458 for (i = 0; i < vsi->num_queue_pairs; i++) {
462 ring = READ_ONCE(vsi->tx_rings[i]);
465 i40e_get_netdev_stats_struct_tx(ring, stats);
467 if (i40e_enabled_xdp_vsi(vsi)) {
468 ring = READ_ONCE(vsi->xdp_rings[i]);
471 i40e_get_netdev_stats_struct_tx(ring, stats);
474 ring = READ_ONCE(vsi->rx_rings[i]);
478 start = u64_stats_fetch_begin_irq(&ring->syncp);
479 packets = ring->stats.packets;
480 bytes = ring->stats.bytes;
481 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
483 stats->rx_packets += packets;
484 stats->rx_bytes += bytes;
489 /* following stats updated by i40e_watchdog_subtask() */
490 stats->multicast = vsi_stats->multicast;
491 stats->tx_errors = vsi_stats->tx_errors;
492 stats->tx_dropped = vsi_stats->tx_dropped;
493 stats->rx_errors = vsi_stats->rx_errors;
494 stats->rx_dropped = vsi_stats->rx_dropped;
495 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
496 stats->rx_length_errors = vsi_stats->rx_length_errors;
500 * i40e_vsi_reset_stats - Resets all stats of the given vsi
501 * @vsi: the VSI to have its stats reset
503 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
505 struct rtnl_link_stats64 *ns;
511 ns = i40e_get_vsi_stats_struct(vsi);
512 memset(ns, 0, sizeof(*ns));
513 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
514 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
515 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
516 if (vsi->rx_rings && vsi->rx_rings[0]) {
517 for (i = 0; i < vsi->num_queue_pairs; i++) {
518 memset(&vsi->rx_rings[i]->stats, 0,
519 sizeof(vsi->rx_rings[i]->stats));
520 memset(&vsi->rx_rings[i]->rx_stats, 0,
521 sizeof(vsi->rx_rings[i]->rx_stats));
522 memset(&vsi->tx_rings[i]->stats, 0,
523 sizeof(vsi->tx_rings[i]->stats));
524 memset(&vsi->tx_rings[i]->tx_stats, 0,
525 sizeof(vsi->tx_rings[i]->tx_stats));
528 vsi->stat_offsets_loaded = false;
532 * i40e_pf_reset_stats - Reset all of the stats for the given PF
533 * @pf: the PF to be reset
535 void i40e_pf_reset_stats(struct i40e_pf *pf)
539 memset(&pf->stats, 0, sizeof(pf->stats));
540 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
541 pf->stat_offsets_loaded = false;
543 for (i = 0; i < I40E_MAX_VEB; i++) {
545 memset(&pf->veb[i]->stats, 0,
546 sizeof(pf->veb[i]->stats));
547 memset(&pf->veb[i]->stats_offsets, 0,
548 sizeof(pf->veb[i]->stats_offsets));
549 memset(&pf->veb[i]->tc_stats, 0,
550 sizeof(pf->veb[i]->tc_stats));
551 memset(&pf->veb[i]->tc_stats_offsets, 0,
552 sizeof(pf->veb[i]->tc_stats_offsets));
553 pf->veb[i]->stat_offsets_loaded = false;
556 pf->hw_csum_rx_error = 0;
560 * i40e_compute_pci_to_hw_id - compute index form PCI function.
561 * @vsi: ptr to the VSI to read from.
562 * @hw: ptr to the hardware info.
564 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
566 int pf_count = i40e_get_pf_count(hw);
568 if (vsi->type == I40E_VSI_SRIOV)
569 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
571 return hw->port + BIT(7);
575 * i40e_stat_update64 - read and update a 64 bit stat from the chip.
576 * @hw: ptr to the hardware info.
577 * @hireg: the high 32 bit reg to read.
578 * @loreg: the low 32 bit reg to read.
579 * @offset_loaded: has the initial offset been loaded yet.
580 * @offset: ptr to current offset value.
581 * @stat: ptr to the stat.
583 * Since the device stats are not reset at PFReset, they will not
584 * be zeroed when the driver starts. We'll save the first values read
585 * and use them as offsets to be subtracted from the raw values in order
586 * to report stats that count from zero.
588 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
589 bool offset_loaded, u64 *offset, u64 *stat)
593 new_data = rd64(hw, loreg);
595 if (!offset_loaded || new_data < *offset)
597 *stat = new_data - *offset;
601 * i40e_stat_update48 - read and update a 48 bit stat from the chip
602 * @hw: ptr to the hardware info
603 * @hireg: the high 32 bit reg to read
604 * @loreg: the low 32 bit reg to read
605 * @offset_loaded: has the initial offset been loaded yet
606 * @offset: ptr to current offset value
607 * @stat: ptr to the stat
609 * Since the device stats are not reset at PFReset, they likely will not
610 * be zeroed when the driver starts. We'll save the first values read
611 * and use them as offsets to be subtracted from the raw values in order
612 * to report stats that count from zero. In the process, we also manage
613 * the potential roll-over.
615 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
616 bool offset_loaded, u64 *offset, u64 *stat)
620 if (hw->device_id == I40E_DEV_ID_QEMU) {
621 new_data = rd32(hw, loreg);
622 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
624 new_data = rd64(hw, loreg);
628 if (likely(new_data >= *offset))
629 *stat = new_data - *offset;
631 *stat = (new_data + BIT_ULL(48)) - *offset;
632 *stat &= 0xFFFFFFFFFFFFULL;
636 * i40e_stat_update32 - read and update a 32 bit stat from the chip
637 * @hw: ptr to the hardware info
638 * @reg: the hw reg to read
639 * @offset_loaded: has the initial offset been loaded yet
640 * @offset: ptr to current offset value
641 * @stat: ptr to the stat
643 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
644 bool offset_loaded, u64 *offset, u64 *stat)
648 new_data = rd32(hw, reg);
651 if (likely(new_data >= *offset))
652 *stat = (u32)(new_data - *offset);
654 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
658 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
659 * @hw: ptr to the hardware info
660 * @reg: the hw reg to read and clear
661 * @stat: ptr to the stat
663 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
665 u32 new_data = rd32(hw, reg);
667 wr32(hw, reg, 1); /* must write a nonzero value to clear register */
672 * i40e_stats_update_rx_discards - update rx_discards.
673 * @vsi: ptr to the VSI to be updated.
674 * @hw: ptr to the hardware info.
675 * @stat_idx: VSI's stat_counter_idx.
676 * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
677 * @stat_offset: ptr to stat_offset to store first read of specific register.
678 * @stat: ptr to VSI's stat to be updated.
681 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
682 int stat_idx, bool offset_loaded,
683 struct i40e_eth_stats *stat_offset,
684 struct i40e_eth_stats *stat)
686 u64 rx_rdpc, rx_rxerr;
688 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
689 &stat_offset->rx_discards, &rx_rdpc);
690 i40e_stat_update64(hw,
691 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
692 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
693 offset_loaded, &stat_offset->rx_discards_other,
696 stat->rx_discards = rx_rdpc + rx_rxerr;
700 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
701 * @vsi: the VSI to be updated
703 void i40e_update_eth_stats(struct i40e_vsi *vsi)
705 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
706 struct i40e_pf *pf = vsi->back;
707 struct i40e_hw *hw = &pf->hw;
708 struct i40e_eth_stats *oes;
709 struct i40e_eth_stats *es; /* device's eth stats */
711 es = &vsi->eth_stats;
712 oes = &vsi->eth_stats_offsets;
714 /* Gather up the stats that the hw collects */
715 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
716 vsi->stat_offsets_loaded,
717 &oes->tx_errors, &es->tx_errors);
718 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
719 vsi->stat_offsets_loaded,
720 &oes->rx_discards, &es->rx_discards);
721 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
722 vsi->stat_offsets_loaded,
723 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
725 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
726 I40E_GLV_GORCL(stat_idx),
727 vsi->stat_offsets_loaded,
728 &oes->rx_bytes, &es->rx_bytes);
729 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
730 I40E_GLV_UPRCL(stat_idx),
731 vsi->stat_offsets_loaded,
732 &oes->rx_unicast, &es->rx_unicast);
733 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
734 I40E_GLV_MPRCL(stat_idx),
735 vsi->stat_offsets_loaded,
736 &oes->rx_multicast, &es->rx_multicast);
737 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
738 I40E_GLV_BPRCL(stat_idx),
739 vsi->stat_offsets_loaded,
740 &oes->rx_broadcast, &es->rx_broadcast);
742 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
743 I40E_GLV_GOTCL(stat_idx),
744 vsi->stat_offsets_loaded,
745 &oes->tx_bytes, &es->tx_bytes);
746 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
747 I40E_GLV_UPTCL(stat_idx),
748 vsi->stat_offsets_loaded,
749 &oes->tx_unicast, &es->tx_unicast);
750 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
751 I40E_GLV_MPTCL(stat_idx),
752 vsi->stat_offsets_loaded,
753 &oes->tx_multicast, &es->tx_multicast);
754 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
755 I40E_GLV_BPTCL(stat_idx),
756 vsi->stat_offsets_loaded,
757 &oes->tx_broadcast, &es->tx_broadcast);
759 i40e_stats_update_rx_discards(vsi, hw, stat_idx,
760 vsi->stat_offsets_loaded, oes, es);
762 vsi->stat_offsets_loaded = true;
766 * i40e_update_veb_stats - Update Switch component statistics
767 * @veb: the VEB being updated
769 void i40e_update_veb_stats(struct i40e_veb *veb)
771 struct i40e_pf *pf = veb->pf;
772 struct i40e_hw *hw = &pf->hw;
773 struct i40e_eth_stats *oes;
774 struct i40e_eth_stats *es; /* device's eth stats */
775 struct i40e_veb_tc_stats *veb_oes;
776 struct i40e_veb_tc_stats *veb_es;
779 idx = veb->stats_idx;
781 oes = &veb->stats_offsets;
782 veb_es = &veb->tc_stats;
783 veb_oes = &veb->tc_stats_offsets;
785 /* Gather up the stats that the hw collects */
786 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
787 veb->stat_offsets_loaded,
788 &oes->tx_discards, &es->tx_discards);
789 if (hw->revision_id > 0)
790 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
791 veb->stat_offsets_loaded,
792 &oes->rx_unknown_protocol,
793 &es->rx_unknown_protocol);
794 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
795 veb->stat_offsets_loaded,
796 &oes->rx_bytes, &es->rx_bytes);
797 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
798 veb->stat_offsets_loaded,
799 &oes->rx_unicast, &es->rx_unicast);
800 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
801 veb->stat_offsets_loaded,
802 &oes->rx_multicast, &es->rx_multicast);
803 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
804 veb->stat_offsets_loaded,
805 &oes->rx_broadcast, &es->rx_broadcast);
807 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
808 veb->stat_offsets_loaded,
809 &oes->tx_bytes, &es->tx_bytes);
810 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
811 veb->stat_offsets_loaded,
812 &oes->tx_unicast, &es->tx_unicast);
813 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
814 veb->stat_offsets_loaded,
815 &oes->tx_multicast, &es->tx_multicast);
816 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
817 veb->stat_offsets_loaded,
818 &oes->tx_broadcast, &es->tx_broadcast);
819 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
820 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
821 I40E_GLVEBTC_RPCL(i, idx),
822 veb->stat_offsets_loaded,
823 &veb_oes->tc_rx_packets[i],
824 &veb_es->tc_rx_packets[i]);
825 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
826 I40E_GLVEBTC_RBCL(i, idx),
827 veb->stat_offsets_loaded,
828 &veb_oes->tc_rx_bytes[i],
829 &veb_es->tc_rx_bytes[i]);
830 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
831 I40E_GLVEBTC_TPCL(i, idx),
832 veb->stat_offsets_loaded,
833 &veb_oes->tc_tx_packets[i],
834 &veb_es->tc_tx_packets[i]);
835 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
836 I40E_GLVEBTC_TBCL(i, idx),
837 veb->stat_offsets_loaded,
838 &veb_oes->tc_tx_bytes[i],
839 &veb_es->tc_tx_bytes[i]);
841 veb->stat_offsets_loaded = true;
845 * i40e_update_vsi_stats - Update the vsi statistics counters.
846 * @vsi: the VSI to be updated
848 * There are a few instances where we store the same stat in a
849 * couple of different structs. This is partly because we have
850 * the netdev stats that need to be filled out, which is slightly
851 * different from the "eth_stats" defined by the chip and used in
852 * VF communications. We sort it out here.
854 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
856 struct i40e_pf *pf = vsi->back;
857 struct rtnl_link_stats64 *ons;
858 struct rtnl_link_stats64 *ns; /* netdev stats */
859 struct i40e_eth_stats *oes;
860 struct i40e_eth_stats *es; /* device's eth stats */
861 u64 tx_restart, tx_busy;
872 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
873 test_bit(__I40E_CONFIG_BUSY, pf->state))
876 ns = i40e_get_vsi_stats_struct(vsi);
877 ons = &vsi->net_stats_offsets;
878 es = &vsi->eth_stats;
879 oes = &vsi->eth_stats_offsets;
881 /* Gather up the netdev and vsi stats that the driver collects
882 * on the fly during packet processing
886 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
890 for (q = 0; q < vsi->num_queue_pairs; q++) {
892 p = READ_ONCE(vsi->tx_rings[q]);
897 start = u64_stats_fetch_begin_irq(&p->syncp);
898 packets = p->stats.packets;
899 bytes = p->stats.bytes;
900 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
903 tx_restart += p->tx_stats.restart_queue;
904 tx_busy += p->tx_stats.tx_busy;
905 tx_linearize += p->tx_stats.tx_linearize;
906 tx_force_wb += p->tx_stats.tx_force_wb;
909 p = READ_ONCE(vsi->rx_rings[q]);
914 start = u64_stats_fetch_begin_irq(&p->syncp);
915 packets = p->stats.packets;
916 bytes = p->stats.bytes;
917 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
920 rx_buf += p->rx_stats.alloc_buff_failed;
921 rx_page += p->rx_stats.alloc_page_failed;
923 if (i40e_enabled_xdp_vsi(vsi)) {
924 /* locate XDP ring */
925 p = READ_ONCE(vsi->xdp_rings[q]);
930 start = u64_stats_fetch_begin_irq(&p->syncp);
931 packets = p->stats.packets;
932 bytes = p->stats.bytes;
933 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
936 tx_restart += p->tx_stats.restart_queue;
937 tx_busy += p->tx_stats.tx_busy;
938 tx_linearize += p->tx_stats.tx_linearize;
939 tx_force_wb += p->tx_stats.tx_force_wb;
943 vsi->tx_restart = tx_restart;
944 vsi->tx_busy = tx_busy;
945 vsi->tx_linearize = tx_linearize;
946 vsi->tx_force_wb = tx_force_wb;
947 vsi->rx_page_failed = rx_page;
948 vsi->rx_buf_failed = rx_buf;
950 ns->rx_packets = rx_p;
952 ns->tx_packets = tx_p;
955 /* update netdev stats from eth stats */
956 i40e_update_eth_stats(vsi);
957 ons->tx_errors = oes->tx_errors;
958 ns->tx_errors = es->tx_errors;
959 ons->multicast = oes->rx_multicast;
960 ns->multicast = es->rx_multicast;
961 ons->rx_dropped = oes->rx_discards;
962 ns->rx_dropped = es->rx_discards;
963 ons->tx_dropped = oes->tx_discards;
964 ns->tx_dropped = es->tx_discards;
966 /* pull in a couple PF stats if this is the main vsi */
967 if (vsi == pf->vsi[pf->lan_vsi]) {
968 ns->rx_crc_errors = pf->stats.crc_errors;
969 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
970 ns->rx_length_errors = pf->stats.rx_length_errors;
975 * i40e_update_pf_stats - Update the PF statistics counters.
976 * @pf: the PF to be updated
978 static void i40e_update_pf_stats(struct i40e_pf *pf)
980 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
981 struct i40e_hw_port_stats *nsd = &pf->stats;
982 struct i40e_hw *hw = &pf->hw;
986 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
987 I40E_GLPRT_GORCL(hw->port),
988 pf->stat_offsets_loaded,
989 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
990 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
991 I40E_GLPRT_GOTCL(hw->port),
992 pf->stat_offsets_loaded,
993 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
994 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
995 pf->stat_offsets_loaded,
996 &osd->eth.rx_discards,
997 &nsd->eth.rx_discards);
998 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
999 I40E_GLPRT_UPRCL(hw->port),
1000 pf->stat_offsets_loaded,
1001 &osd->eth.rx_unicast,
1002 &nsd->eth.rx_unicast);
1003 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1004 I40E_GLPRT_MPRCL(hw->port),
1005 pf->stat_offsets_loaded,
1006 &osd->eth.rx_multicast,
1007 &nsd->eth.rx_multicast);
1008 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1009 I40E_GLPRT_BPRCL(hw->port),
1010 pf->stat_offsets_loaded,
1011 &osd->eth.rx_broadcast,
1012 &nsd->eth.rx_broadcast);
1013 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1014 I40E_GLPRT_UPTCL(hw->port),
1015 pf->stat_offsets_loaded,
1016 &osd->eth.tx_unicast,
1017 &nsd->eth.tx_unicast);
1018 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1019 I40E_GLPRT_MPTCL(hw->port),
1020 pf->stat_offsets_loaded,
1021 &osd->eth.tx_multicast,
1022 &nsd->eth.tx_multicast);
1023 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1024 I40E_GLPRT_BPTCL(hw->port),
1025 pf->stat_offsets_loaded,
1026 &osd->eth.tx_broadcast,
1027 &nsd->eth.tx_broadcast);
1029 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1030 pf->stat_offsets_loaded,
1031 &osd->tx_dropped_link_down,
1032 &nsd->tx_dropped_link_down);
1034 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1035 pf->stat_offsets_loaded,
1036 &osd->crc_errors, &nsd->crc_errors);
1038 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1039 pf->stat_offsets_loaded,
1040 &osd->illegal_bytes, &nsd->illegal_bytes);
1042 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1043 pf->stat_offsets_loaded,
1044 &osd->mac_local_faults,
1045 &nsd->mac_local_faults);
1046 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1047 pf->stat_offsets_loaded,
1048 &osd->mac_remote_faults,
1049 &nsd->mac_remote_faults);
1051 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1052 pf->stat_offsets_loaded,
1053 &osd->rx_length_errors,
1054 &nsd->rx_length_errors);
1056 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1057 pf->stat_offsets_loaded,
1058 &osd->link_xon_rx, &nsd->link_xon_rx);
1059 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1060 pf->stat_offsets_loaded,
1061 &osd->link_xon_tx, &nsd->link_xon_tx);
1062 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1063 pf->stat_offsets_loaded,
1064 &osd->link_xoff_rx, &nsd->link_xoff_rx);
1065 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1066 pf->stat_offsets_loaded,
1067 &osd->link_xoff_tx, &nsd->link_xoff_tx);
1069 for (i = 0; i < 8; i++) {
1070 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1071 pf->stat_offsets_loaded,
1072 &osd->priority_xoff_rx[i],
1073 &nsd->priority_xoff_rx[i]);
1074 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1075 pf->stat_offsets_loaded,
1076 &osd->priority_xon_rx[i],
1077 &nsd->priority_xon_rx[i]);
1078 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1079 pf->stat_offsets_loaded,
1080 &osd->priority_xon_tx[i],
1081 &nsd->priority_xon_tx[i]);
1082 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1083 pf->stat_offsets_loaded,
1084 &osd->priority_xoff_tx[i],
1085 &nsd->priority_xoff_tx[i]);
1086 i40e_stat_update32(hw,
1087 I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1088 pf->stat_offsets_loaded,
1089 &osd->priority_xon_2_xoff[i],
1090 &nsd->priority_xon_2_xoff[i]);
1093 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1094 I40E_GLPRT_PRC64L(hw->port),
1095 pf->stat_offsets_loaded,
1096 &osd->rx_size_64, &nsd->rx_size_64);
1097 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1098 I40E_GLPRT_PRC127L(hw->port),
1099 pf->stat_offsets_loaded,
1100 &osd->rx_size_127, &nsd->rx_size_127);
1101 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1102 I40E_GLPRT_PRC255L(hw->port),
1103 pf->stat_offsets_loaded,
1104 &osd->rx_size_255, &nsd->rx_size_255);
1105 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1106 I40E_GLPRT_PRC511L(hw->port),
1107 pf->stat_offsets_loaded,
1108 &osd->rx_size_511, &nsd->rx_size_511);
1109 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1110 I40E_GLPRT_PRC1023L(hw->port),
1111 pf->stat_offsets_loaded,
1112 &osd->rx_size_1023, &nsd->rx_size_1023);
1113 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1114 I40E_GLPRT_PRC1522L(hw->port),
1115 pf->stat_offsets_loaded,
1116 &osd->rx_size_1522, &nsd->rx_size_1522);
1117 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1118 I40E_GLPRT_PRC9522L(hw->port),
1119 pf->stat_offsets_loaded,
1120 &osd->rx_size_big, &nsd->rx_size_big);
1122 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1123 I40E_GLPRT_PTC64L(hw->port),
1124 pf->stat_offsets_loaded,
1125 &osd->tx_size_64, &nsd->tx_size_64);
1126 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1127 I40E_GLPRT_PTC127L(hw->port),
1128 pf->stat_offsets_loaded,
1129 &osd->tx_size_127, &nsd->tx_size_127);
1130 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1131 I40E_GLPRT_PTC255L(hw->port),
1132 pf->stat_offsets_loaded,
1133 &osd->tx_size_255, &nsd->tx_size_255);
1134 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1135 I40E_GLPRT_PTC511L(hw->port),
1136 pf->stat_offsets_loaded,
1137 &osd->tx_size_511, &nsd->tx_size_511);
1138 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1139 I40E_GLPRT_PTC1023L(hw->port),
1140 pf->stat_offsets_loaded,
1141 &osd->tx_size_1023, &nsd->tx_size_1023);
1142 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1143 I40E_GLPRT_PTC1522L(hw->port),
1144 pf->stat_offsets_loaded,
1145 &osd->tx_size_1522, &nsd->tx_size_1522);
1146 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1147 I40E_GLPRT_PTC9522L(hw->port),
1148 pf->stat_offsets_loaded,
1149 &osd->tx_size_big, &nsd->tx_size_big);
1151 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1152 pf->stat_offsets_loaded,
1153 &osd->rx_undersize, &nsd->rx_undersize);
1154 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1155 pf->stat_offsets_loaded,
1156 &osd->rx_fragments, &nsd->rx_fragments);
1157 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1158 pf->stat_offsets_loaded,
1159 &osd->rx_oversize, &nsd->rx_oversize);
1160 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1161 pf->stat_offsets_loaded,
1162 &osd->rx_jabber, &nsd->rx_jabber);
1165 i40e_stat_update_and_clear32(hw,
1166 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1167 &nsd->fd_atr_match);
1168 i40e_stat_update_and_clear32(hw,
1169 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1171 i40e_stat_update_and_clear32(hw,
1172 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1173 &nsd->fd_atr_tunnel_match);
1175 val = rd32(hw, I40E_PRTPM_EEE_STAT);
1176 nsd->tx_lpi_status =
1177 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1178 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1179 nsd->rx_lpi_status =
1180 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1181 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1182 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1183 pf->stat_offsets_loaded,
1184 &osd->tx_lpi_count, &nsd->tx_lpi_count);
1185 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1186 pf->stat_offsets_loaded,
1187 &osd->rx_lpi_count, &nsd->rx_lpi_count);
1189 if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1190 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1191 nsd->fd_sb_status = true;
1193 nsd->fd_sb_status = false;
1195 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1196 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1197 nsd->fd_atr_status = true;
1199 nsd->fd_atr_status = false;
1201 pf->stat_offsets_loaded = true;
1205 * i40e_update_stats - Update the various statistics counters.
1206 * @vsi: the VSI to be updated
1208 * Update the various stats for this VSI and its related entities.
1210 void i40e_update_stats(struct i40e_vsi *vsi)
1212 struct i40e_pf *pf = vsi->back;
1214 if (vsi == pf->vsi[pf->lan_vsi])
1215 i40e_update_pf_stats(pf);
1217 i40e_update_vsi_stats(vsi);
1221 * i40e_count_filters - counts VSI mac filters
1222 * @vsi: the VSI to be searched
1224 * Returns count of mac filters
1226 int i40e_count_filters(struct i40e_vsi *vsi)
1228 struct i40e_mac_filter *f;
1229 struct hlist_node *h;
1233 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1234 if (f->state == I40E_FILTER_NEW ||
1235 f->state == I40E_FILTER_ACTIVE)
1243 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1244 * @vsi: the VSI to be searched
1245 * @macaddr: the MAC address
1248 * Returns ptr to the filter object or NULL
1250 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1251 const u8 *macaddr, s16 vlan)
1253 struct i40e_mac_filter *f;
1256 if (!vsi || !macaddr)
1259 key = i40e_addr_to_hkey(macaddr);
1260 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1261 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1269 * i40e_find_mac - Find a mac addr in the macvlan filters list
1270 * @vsi: the VSI to be searched
1271 * @macaddr: the MAC address we are searching for
1273 * Returns the first filter with the provided MAC address or NULL if
1274 * MAC address was not found
1276 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1278 struct i40e_mac_filter *f;
1281 if (!vsi || !macaddr)
1284 key = i40e_addr_to_hkey(macaddr);
1285 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1286 if ((ether_addr_equal(macaddr, f->macaddr)))
1293 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1294 * @vsi: the VSI to be searched
1296 * Returns true if VSI is in vlan mode or false otherwise
1298 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1300 /* If we have a PVID, always operate in VLAN mode */
1304 /* We need to operate in VLAN mode whenever we have any filters with
1305 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1306 * time, incurring search cost repeatedly. However, we can notice two
1309 * 1) the only place where we can gain a VLAN filter is in
1312 * 2) the only place where filters are actually removed is in
1313 * i40e_sync_filters_subtask.
1315 * Thus, we can simply use a boolean value, has_vlan_filters which we
1316 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1317 * we have to perform the full search after deleting filters in
1318 * i40e_sync_filters_subtask, but we already have to search
1319 * filters here and can perform the check at the same time. This
1320 * results in avoiding embedding a loop for VLAN mode inside another
1321 * loop over all the filters, and should maintain correctness as noted
1324 return vsi->has_vlan_filter;
1328 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1329 * @vsi: the VSI to configure
1330 * @tmp_add_list: list of filters ready to be added
1331 * @tmp_del_list: list of filters ready to be deleted
1332 * @vlan_filters: the number of active VLAN filters
1334 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1335 * behave as expected. If we have any active VLAN filters remaining or about
1336 * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1337 * so that they only match against untagged traffic. If we no longer have any
1338 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1339 * so that they match against both tagged and untagged traffic. In this way,
1340 * we ensure that we correctly receive the desired traffic. This ensures that
1341 * when we have an active VLAN we will receive only untagged traffic and
1342 * traffic matching active VLANs. If we have no active VLANs then we will
1343 * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1345 * Finally, in a similar fashion, this function also corrects filters when
1346 * there is an active PVID assigned to this VSI.
1348 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1350 * This function is only expected to be called from within
1351 * i40e_sync_vsi_filters.
1353 * NOTE: This function expects to be called while under the
1354 * mac_filter_hash_lock
1356 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1357 struct hlist_head *tmp_add_list,
1358 struct hlist_head *tmp_del_list,
1361 s16 pvid = le16_to_cpu(vsi->info.pvid);
1362 struct i40e_mac_filter *f, *add_head;
1363 struct i40e_new_mac_filter *new;
1364 struct hlist_node *h;
1367 /* To determine if a particular filter needs to be replaced we
1368 * have the three following conditions:
1370 * a) if we have a PVID assigned, then all filters which are
1371 * not marked as VLAN=PVID must be replaced with filters that
1373 * b) otherwise, if we have any active VLANS, all filters
1374 * which are marked as VLAN=-1 must be replaced with
1375 * filters marked as VLAN=0
1376 * c) finally, if we do not have any active VLANS, all filters
1377 * which are marked as VLAN=0 must be replaced with filters
1381 /* Update the filters about to be added in place */
1382 hlist_for_each_entry(new, tmp_add_list, hlist) {
1383 if (pvid && new->f->vlan != pvid)
1384 new->f->vlan = pvid;
1385 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1387 else if (!vlan_filters && new->f->vlan == 0)
1388 new->f->vlan = I40E_VLAN_ANY;
1391 /* Update the remaining active filters */
1392 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1393 /* Combine the checks for whether a filter needs to be changed
1394 * and then determine the new VLAN inside the if block, in
1395 * order to avoid duplicating code for adding the new filter
1396 * then deleting the old filter.
1398 if ((pvid && f->vlan != pvid) ||
1399 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1400 (!vlan_filters && f->vlan == 0)) {
1401 /* Determine the new vlan we will be adding */
1404 else if (vlan_filters)
1407 new_vlan = I40E_VLAN_ANY;
1409 /* Create the new filter */
1410 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1414 /* Create a temporary i40e_new_mac_filter */
1415 new = kzalloc(sizeof(*new), GFP_ATOMIC);
1420 new->state = add_head->state;
1422 /* Add the new filter to the tmp list */
1423 hlist_add_head(&new->hlist, tmp_add_list);
1425 /* Put the original filter into the delete list */
1426 f->state = I40E_FILTER_REMOVE;
1427 hash_del(&f->hlist);
1428 hlist_add_head(&f->hlist, tmp_del_list);
1432 vsi->has_vlan_filter = !!vlan_filters;
1438 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1439 * @vsi: the PF Main VSI - inappropriate for any other VSI
1440 * @macaddr: the MAC address
1442 * Remove whatever filter the firmware set up so the driver can manage
1443 * its own filtering intelligently.
1445 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1447 struct i40e_aqc_remove_macvlan_element_data element;
1448 struct i40e_pf *pf = vsi->back;
1450 /* Only appropriate for the PF main VSI */
1451 if (vsi->type != I40E_VSI_MAIN)
1454 memset(&element, 0, sizeof(element));
1455 ether_addr_copy(element.mac_addr, macaddr);
1456 element.vlan_tag = 0;
1457 /* Ignore error returns, some firmware does it this way... */
1458 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1459 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1461 memset(&element, 0, sizeof(element));
1462 ether_addr_copy(element.mac_addr, macaddr);
1463 element.vlan_tag = 0;
1464 /* ...and some firmware does it this way. */
1465 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1466 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1467 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1471 * i40e_add_filter - Add a mac/vlan filter to the VSI
1472 * @vsi: the VSI to be searched
1473 * @macaddr: the MAC address
1476 * Returns ptr to the filter object or NULL when no memory available.
1478 * NOTE: This function is expected to be called with mac_filter_hash_lock
1481 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1482 const u8 *macaddr, s16 vlan)
1484 struct i40e_mac_filter *f;
1487 if (!vsi || !macaddr)
1490 f = i40e_find_filter(vsi, macaddr, vlan);
1492 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1496 /* Update the boolean indicating if we need to function in
1500 vsi->has_vlan_filter = true;
1502 ether_addr_copy(f->macaddr, macaddr);
1504 f->state = I40E_FILTER_NEW;
1505 INIT_HLIST_NODE(&f->hlist);
1507 key = i40e_addr_to_hkey(macaddr);
1508 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1510 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1511 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1514 /* If we're asked to add a filter that has been marked for removal, it
1515 * is safe to simply restore it to active state. __i40e_del_filter
1516 * will have simply deleted any filters which were previously marked
1517 * NEW or FAILED, so if it is currently marked REMOVE it must have
1518 * previously been ACTIVE. Since we haven't yet run the sync filters
1519 * task, just restore this filter to the ACTIVE state so that the
1520 * sync task leaves it in place
1522 if (f->state == I40E_FILTER_REMOVE)
1523 f->state = I40E_FILTER_ACTIVE;
1529 * __i40e_del_filter - Remove a specific filter from the VSI
1530 * @vsi: VSI to remove from
1531 * @f: the filter to remove from the list
1533 * This function should be called instead of i40e_del_filter only if you know
1534 * the exact filter you will remove already, such as via i40e_find_filter or
1537 * NOTE: This function is expected to be called with mac_filter_hash_lock
1539 * ANOTHER NOTE: This function MUST be called from within the context of
1540 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1541 * instead of list_for_each_entry().
1543 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1548 /* If the filter was never added to firmware then we can just delete it
1549 * directly and we don't want to set the status to remove or else an
1550 * admin queue command will unnecessarily fire.
1552 if ((f->state == I40E_FILTER_FAILED) ||
1553 (f->state == I40E_FILTER_NEW)) {
1554 hash_del(&f->hlist);
1557 f->state = I40E_FILTER_REMOVE;
1560 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1561 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1565 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1566 * @vsi: the VSI to be searched
1567 * @macaddr: the MAC address
1570 * NOTE: This function is expected to be called with mac_filter_hash_lock
1572 * ANOTHER NOTE: This function MUST be called from within the context of
1573 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1574 * instead of list_for_each_entry().
1576 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1578 struct i40e_mac_filter *f;
1580 if (!vsi || !macaddr)
1583 f = i40e_find_filter(vsi, macaddr, vlan);
1584 __i40e_del_filter(vsi, f);
1588 * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1589 * @vsi: the VSI to be searched
1590 * @macaddr: the mac address to be filtered
1592 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1593 * go through all the macvlan filters and add a macvlan filter for each
1594 * unique vlan that already exists. If a PVID has been assigned, instead only
1595 * add the macaddr to that VLAN.
1597 * Returns last filter added on success, else NULL
1599 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1602 struct i40e_mac_filter *f, *add = NULL;
1603 struct hlist_node *h;
1607 return i40e_add_filter(vsi, macaddr,
1608 le16_to_cpu(vsi->info.pvid));
1610 if (!i40e_is_vsi_in_vlan(vsi))
1611 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1613 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1614 if (f->state == I40E_FILTER_REMOVE)
1616 add = i40e_add_filter(vsi, macaddr, f->vlan);
1625 * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1626 * @vsi: the VSI to be searched
1627 * @macaddr: the mac address to be removed
1629 * Removes a given MAC address from a VSI regardless of what VLAN it has been
1632 * Returns 0 for success, or error
1634 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1636 struct i40e_mac_filter *f;
1637 struct hlist_node *h;
1641 lockdep_assert_held(&vsi->mac_filter_hash_lock);
1642 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1643 if (ether_addr_equal(macaddr, f->macaddr)) {
1644 __i40e_del_filter(vsi, f);
1656 * i40e_set_mac - NDO callback to set mac address
1657 * @netdev: network interface device structure
1658 * @p: pointer to an address structure
1660 * Returns 0 on success, negative on failure
1662 static int i40e_set_mac(struct net_device *netdev, void *p)
1664 struct i40e_netdev_priv *np = netdev_priv(netdev);
1665 struct i40e_vsi *vsi = np->vsi;
1666 struct i40e_pf *pf = vsi->back;
1667 struct i40e_hw *hw = &pf->hw;
1668 struct sockaddr *addr = p;
1670 if (!is_valid_ether_addr(addr->sa_data))
1671 return -EADDRNOTAVAIL;
1673 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1674 netdev_info(netdev, "already using mac address %pM\n",
1679 if (test_bit(__I40E_DOWN, pf->state) ||
1680 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1681 return -EADDRNOTAVAIL;
1683 if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1684 netdev_info(netdev, "returning to hw mac address %pM\n",
1687 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1689 /* Copy the address first, so that we avoid a possible race with
1691 * - Remove old address from MAC filter
1692 * - Copy new address
1693 * - Add new address to MAC filter
1695 spin_lock_bh(&vsi->mac_filter_hash_lock);
1696 i40e_del_mac_filter(vsi, netdev->dev_addr);
1697 ether_addr_copy(netdev->dev_addr, addr->sa_data);
1698 i40e_add_mac_filter(vsi, netdev->dev_addr);
1699 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1701 if (vsi->type == I40E_VSI_MAIN) {
1704 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1705 addr->sa_data, NULL);
1707 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1708 i40e_stat_str(hw, ret),
1709 i40e_aq_str(hw, hw->aq.asq_last_status));
1712 /* schedule our worker thread which will take care of
1713 * applying the new filter changes
1715 i40e_service_event_schedule(pf);
1720 * i40e_config_rss_aq - Prepare for RSS using AQ commands
1721 * @vsi: vsi structure
1722 * @seed: RSS hash seed
1723 * @lut: pointer to lookup table of lut_size
1724 * @lut_size: size of the lookup table
1726 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1727 u8 *lut, u16 lut_size)
1729 struct i40e_pf *pf = vsi->back;
1730 struct i40e_hw *hw = &pf->hw;
1734 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1735 (struct i40e_aqc_get_set_rss_key_data *)seed;
1736 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1738 dev_info(&pf->pdev->dev,
1739 "Cannot set RSS key, err %s aq_err %s\n",
1740 i40e_stat_str(hw, ret),
1741 i40e_aq_str(hw, hw->aq.asq_last_status));
1746 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1748 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1750 dev_info(&pf->pdev->dev,
1751 "Cannot set RSS lut, err %s aq_err %s\n",
1752 i40e_stat_str(hw, ret),
1753 i40e_aq_str(hw, hw->aq.asq_last_status));
1761 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1762 * @vsi: VSI structure
1764 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1766 struct i40e_pf *pf = vsi->back;
1767 u8 seed[I40E_HKEY_ARRAY_SIZE];
1771 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1774 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1775 vsi->num_queue_pairs);
1778 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1782 /* Use the user configured hash keys and lookup table if there is one,
1783 * otherwise use default
1785 if (vsi->rss_lut_user)
1786 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1788 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1789 if (vsi->rss_hkey_user)
1790 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1792 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1793 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1799 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1800 * @vsi: the VSI being configured,
1801 * @ctxt: VSI context structure
1802 * @enabled_tc: number of traffic classes to enable
1804 * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1806 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1807 struct i40e_vsi_context *ctxt,
1810 u16 qcount = 0, max_qcount, qmap, sections = 0;
1811 int i, override_q, pow, num_qps, ret;
1812 u8 netdev_tc = 0, offset = 0;
1814 if (vsi->type != I40E_VSI_MAIN)
1816 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1817 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1818 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1819 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1820 num_qps = vsi->mqprio_qopt.qopt.count[0];
1822 /* find the next higher power-of-2 of num queue pairs */
1823 pow = ilog2(num_qps);
1824 if (!is_power_of_2(num_qps))
1826 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1827 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1829 /* Setup queue offset/count for all TCs for given VSI */
1830 max_qcount = vsi->mqprio_qopt.qopt.count[0];
1831 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1832 /* See if the given TC is enabled for the given VSI */
1833 if (vsi->tc_config.enabled_tc & BIT(i)) {
1834 offset = vsi->mqprio_qopt.qopt.offset[i];
1835 qcount = vsi->mqprio_qopt.qopt.count[i];
1836 if (qcount > max_qcount)
1837 max_qcount = qcount;
1838 vsi->tc_config.tc_info[i].qoffset = offset;
1839 vsi->tc_config.tc_info[i].qcount = qcount;
1840 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1842 /* TC is not enabled so set the offset to
1843 * default queue and allocate one queue
1846 vsi->tc_config.tc_info[i].qoffset = 0;
1847 vsi->tc_config.tc_info[i].qcount = 1;
1848 vsi->tc_config.tc_info[i].netdev_tc = 0;
1852 /* Set actual Tx/Rx queue pairs */
1853 vsi->num_queue_pairs = offset + qcount;
1855 /* Setup queue TC[0].qmap for given VSI context */
1856 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1857 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1858 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1859 ctxt->info.valid_sections |= cpu_to_le16(sections);
1861 /* Reconfigure RSS for main VSI with max queue count */
1862 vsi->rss_size = max_qcount;
1863 ret = i40e_vsi_config_rss(vsi);
1865 dev_info(&vsi->back->pdev->dev,
1866 "Failed to reconfig rss for num_queues (%u)\n",
1870 vsi->reconfig_rss = true;
1871 dev_dbg(&vsi->back->pdev->dev,
1872 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1874 /* Find queue count available for channel VSIs and starting offset
1877 override_q = vsi->mqprio_qopt.qopt.count[0];
1878 if (override_q && override_q < vsi->num_queue_pairs) {
1879 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1880 vsi->next_base_queue = override_q;
1886 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1887 * @vsi: the VSI being setup
1888 * @ctxt: VSI context structure
1889 * @enabled_tc: Enabled TCs bitmap
1890 * @is_add: True if called before Add VSI
1892 * Setup VSI queue mapping for enabled traffic classes.
1894 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1895 struct i40e_vsi_context *ctxt,
1899 struct i40e_pf *pf = vsi->back;
1909 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1911 /* zero out queue mapping, it will get updated on the end of the function */
1912 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
1914 if (vsi->type == I40E_VSI_MAIN) {
1915 /* This code helps add more queue to the VSI if we have
1916 * more cores than RSS can support, the higher cores will
1917 * be served by ATR or other filters. Furthermore, the
1918 * non-zero req_queue_pairs says that user requested a new
1919 * queue count via ethtool's set_channels, so use this
1920 * value for queues distribution across traffic classes
1921 * We need at least one queue pair for the interface
1922 * to be usable as we see in else statement.
1924 if (vsi->req_queue_pairs > 0)
1925 vsi->num_queue_pairs = vsi->req_queue_pairs;
1926 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1927 vsi->num_queue_pairs = pf->num_lan_msix;
1929 vsi->num_queue_pairs = 1;
1932 /* Number of queues per enabled TC */
1933 if (vsi->type == I40E_VSI_MAIN ||
1934 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
1935 num_tc_qps = vsi->num_queue_pairs;
1937 num_tc_qps = vsi->alloc_queue_pairs;
1939 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1940 /* Find numtc from enabled TC bitmap */
1941 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1942 if (enabled_tc & BIT(i)) /* TC is enabled */
1946 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1949 num_tc_qps = num_tc_qps / numtc;
1950 num_tc_qps = min_t(int, num_tc_qps,
1951 i40e_pf_get_max_q_per_tc(pf));
1954 vsi->tc_config.numtc = numtc;
1955 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1957 /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1958 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1959 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1961 /* Setup queue offset/count for all TCs for given VSI */
1962 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1963 /* See if the given TC is enabled for the given VSI */
1964 if (vsi->tc_config.enabled_tc & BIT(i)) {
1968 switch (vsi->type) {
1970 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1971 I40E_FLAG_FD_ATR_ENABLED)) ||
1972 vsi->tc_config.enabled_tc != 1) {
1973 qcount = min_t(int, pf->alloc_rss_size,
1979 case I40E_VSI_SRIOV:
1980 case I40E_VSI_VMDQ2:
1982 qcount = num_tc_qps;
1986 vsi->tc_config.tc_info[i].qoffset = offset;
1987 vsi->tc_config.tc_info[i].qcount = qcount;
1989 /* find the next higher power-of-2 of num queue pairs */
1992 while (num_qps && (BIT_ULL(pow) < qcount)) {
1997 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1999 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2000 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2004 /* TC is not enabled so set the offset to
2005 * default queue and allocate one queue
2008 vsi->tc_config.tc_info[i].qoffset = 0;
2009 vsi->tc_config.tc_info[i].qcount = 1;
2010 vsi->tc_config.tc_info[i].netdev_tc = 0;
2014 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2016 /* Do not change previously set num_queue_pairs for PFs and VFs*/
2017 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2018 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2019 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2020 vsi->num_queue_pairs = offset;
2022 /* Scheduler section valid can only be set for ADD VSI */
2024 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2026 ctxt->info.up_enable_bits = enabled_tc;
2028 if (vsi->type == I40E_VSI_SRIOV) {
2029 ctxt->info.mapping_flags |=
2030 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2031 for (i = 0; i < vsi->num_queue_pairs; i++)
2032 ctxt->info.queue_mapping[i] =
2033 cpu_to_le16(vsi->base_queue + i);
2035 ctxt->info.mapping_flags |=
2036 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2037 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2039 ctxt->info.valid_sections |= cpu_to_le16(sections);
2043 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2044 * @netdev: the netdevice
2045 * @addr: address to add
2047 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2048 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2050 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2052 struct i40e_netdev_priv *np = netdev_priv(netdev);
2053 struct i40e_vsi *vsi = np->vsi;
2055 if (i40e_add_mac_filter(vsi, addr))
2062 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2063 * @netdev: the netdevice
2064 * @addr: address to add
2066 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2067 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2069 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2071 struct i40e_netdev_priv *np = netdev_priv(netdev);
2072 struct i40e_vsi *vsi = np->vsi;
2074 /* Under some circumstances, we might receive a request to delete
2075 * our own device address from our uc list. Because we store the
2076 * device address in the VSI's MAC/VLAN filter list, we need to ignore
2077 * such requests and not delete our device address from this list.
2079 if (ether_addr_equal(addr, netdev->dev_addr))
2082 i40e_del_mac_filter(vsi, addr);
2088 * i40e_set_rx_mode - NDO callback to set the netdev filters
2089 * @netdev: network interface device structure
2091 static void i40e_set_rx_mode(struct net_device *netdev)
2093 struct i40e_netdev_priv *np = netdev_priv(netdev);
2094 struct i40e_vsi *vsi = np->vsi;
2096 spin_lock_bh(&vsi->mac_filter_hash_lock);
2098 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2099 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2101 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2103 /* check for other flag changes */
2104 if (vsi->current_netdev_flags != vsi->netdev->flags) {
2105 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2106 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2111 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2112 * @vsi: Pointer to VSI struct
2113 * @from: Pointer to list which contains MAC filter entries - changes to
2114 * those entries needs to be undone.
2116 * MAC filter entries from this list were slated for deletion.
2118 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2119 struct hlist_head *from)
2121 struct i40e_mac_filter *f;
2122 struct hlist_node *h;
2124 hlist_for_each_entry_safe(f, h, from, hlist) {
2125 u64 key = i40e_addr_to_hkey(f->macaddr);
2127 /* Move the element back into MAC filter list*/
2128 hlist_del(&f->hlist);
2129 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2134 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2135 * @vsi: Pointer to vsi struct
2136 * @from: Pointer to list which contains MAC filter entries - changes to
2137 * those entries needs to be undone.
2139 * MAC filter entries from this list were slated for addition.
2141 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2142 struct hlist_head *from)
2144 struct i40e_new_mac_filter *new;
2145 struct hlist_node *h;
2147 hlist_for_each_entry_safe(new, h, from, hlist) {
2148 /* We can simply free the wrapper structure */
2149 hlist_del(&new->hlist);
2150 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2156 * i40e_next_entry - Get the next non-broadcast filter from a list
2157 * @next: pointer to filter in list
2159 * Returns the next non-broadcast filter in the list. Required so that we
2160 * ignore broadcast filters within the list, since these are not handled via
2161 * the normal firmware update path.
2164 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2166 hlist_for_each_entry_continue(next, hlist) {
2167 if (!is_broadcast_ether_addr(next->f->macaddr))
2175 * i40e_update_filter_state - Update filter state based on return data
2177 * @count: Number of filters added
2178 * @add_list: return data from fw
2179 * @add_head: pointer to first filter in current batch
2181 * MAC filter entries from list were slated to be added to device. Returns
2182 * number of successful filters. Note that 0 does NOT mean success!
2185 i40e_update_filter_state(int count,
2186 struct i40e_aqc_add_macvlan_element_data *add_list,
2187 struct i40e_new_mac_filter *add_head)
2192 for (i = 0; i < count; i++) {
2193 /* Always check status of each filter. We don't need to check
2194 * the firmware return status because we pre-set the filter
2195 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2196 * request to the adminq. Thus, if it no longer matches then
2197 * we know the filter is active.
2199 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2200 add_head->state = I40E_FILTER_FAILED;
2202 add_head->state = I40E_FILTER_ACTIVE;
2206 add_head = i40e_next_filter(add_head);
2215 * i40e_aqc_del_filters - Request firmware to delete a set of filters
2216 * @vsi: ptr to the VSI
2217 * @vsi_name: name to display in messages
2218 * @list: the list of filters to send to firmware
2219 * @num_del: the number of filters to delete
2220 * @retval: Set to -EIO on failure to delete
2222 * Send a request to firmware via AdminQ to delete a set of filters. Uses
2223 * *retval instead of a return value so that success does not force ret_val to
2224 * be set to 0. This ensures that a sequence of calls to this function
2225 * preserve the previous value of *retval on successful delete.
2228 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2229 struct i40e_aqc_remove_macvlan_element_data *list,
2230 int num_del, int *retval)
2232 struct i40e_hw *hw = &vsi->back->hw;
2236 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2237 aq_err = hw->aq.asq_last_status;
2239 /* Explicitly ignore and do not report when firmware returns ENOENT */
2240 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2242 dev_info(&vsi->back->pdev->dev,
2243 "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2244 vsi_name, i40e_stat_str(hw, aq_ret),
2245 i40e_aq_str(hw, aq_err));
2250 * i40e_aqc_add_filters - Request firmware to add a set of filters
2251 * @vsi: ptr to the VSI
2252 * @vsi_name: name to display in messages
2253 * @list: the list of filters to send to firmware
2254 * @add_head: Position in the add hlist
2255 * @num_add: the number of filters to add
2257 * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2258 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2259 * space for more filters.
2262 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2263 struct i40e_aqc_add_macvlan_element_data *list,
2264 struct i40e_new_mac_filter *add_head,
2267 struct i40e_hw *hw = &vsi->back->hw;
2270 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2271 aq_err = hw->aq.asq_last_status;
2272 fcnt = i40e_update_filter_state(num_add, list, add_head);
2274 if (fcnt != num_add) {
2275 if (vsi->type == I40E_VSI_MAIN) {
2276 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2277 dev_warn(&vsi->back->pdev->dev,
2278 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2279 i40e_aq_str(hw, aq_err), vsi_name);
2280 } else if (vsi->type == I40E_VSI_SRIOV ||
2281 vsi->type == I40E_VSI_VMDQ1 ||
2282 vsi->type == I40E_VSI_VMDQ2) {
2283 dev_warn(&vsi->back->pdev->dev,
2284 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2285 i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2287 dev_warn(&vsi->back->pdev->dev,
2288 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2289 i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2295 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2296 * @vsi: pointer to the VSI
2297 * @vsi_name: the VSI name
2300 * This function sets or clears the promiscuous broadcast flags for VLAN
2301 * filters in order to properly receive broadcast frames. Assumes that only
2302 * broadcast filters are passed.
2304 * Returns status indicating success or failure;
2307 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2308 struct i40e_mac_filter *f)
2310 bool enable = f->state == I40E_FILTER_NEW;
2311 struct i40e_hw *hw = &vsi->back->hw;
2314 if (f->vlan == I40E_VLAN_ANY) {
2315 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2320 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2328 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2329 dev_warn(&vsi->back->pdev->dev,
2330 "Error %s, forcing overflow promiscuous on %s\n",
2331 i40e_aq_str(hw, hw->aq.asq_last_status),
2339 * i40e_set_promiscuous - set promiscuous mode
2340 * @pf: board private structure
2341 * @promisc: promisc on or off
2343 * There are different ways of setting promiscuous mode on a PF depending on
2344 * what state/environment we're in. This identifies and sets it appropriately.
2345 * Returns 0 on success.
2347 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2349 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2350 struct i40e_hw *hw = &pf->hw;
2353 if (vsi->type == I40E_VSI_MAIN &&
2354 pf->lan_veb != I40E_NO_VEB &&
2355 !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2356 /* set defport ON for Main VSI instead of true promisc
2357 * this way we will get all unicast/multicast and VLAN
2358 * promisc behavior but will not get VF or VMDq traffic
2359 * replicated on the Main VSI.
2362 aq_ret = i40e_aq_set_default_vsi(hw,
2366 aq_ret = i40e_aq_clear_default_vsi(hw,
2370 dev_info(&pf->pdev->dev,
2371 "Set default VSI failed, err %s, aq_err %s\n",
2372 i40e_stat_str(hw, aq_ret),
2373 i40e_aq_str(hw, hw->aq.asq_last_status));
2376 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2382 dev_info(&pf->pdev->dev,
2383 "set unicast promisc failed, err %s, aq_err %s\n",
2384 i40e_stat_str(hw, aq_ret),
2385 i40e_aq_str(hw, hw->aq.asq_last_status));
2387 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2392 dev_info(&pf->pdev->dev,
2393 "set multicast promisc failed, err %s, aq_err %s\n",
2394 i40e_stat_str(hw, aq_ret),
2395 i40e_aq_str(hw, hw->aq.asq_last_status));
2400 pf->cur_promisc = promisc;
2406 * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2407 * @vsi: ptr to the VSI
2409 * Push any outstanding VSI filter changes through the AdminQ.
2411 * Returns 0 or error value
2413 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2415 struct hlist_head tmp_add_list, tmp_del_list;
2416 struct i40e_mac_filter *f;
2417 struct i40e_new_mac_filter *new, *add_head = NULL;
2418 struct i40e_hw *hw = &vsi->back->hw;
2419 bool old_overflow, new_overflow;
2420 unsigned int failed_filters = 0;
2421 unsigned int vlan_filters = 0;
2422 char vsi_name[16] = "PF";
2423 int filter_list_len = 0;
2424 i40e_status aq_ret = 0;
2425 u32 changed_flags = 0;
2426 struct hlist_node *h;
2435 /* empty array typed pointers, kcalloc later */
2436 struct i40e_aqc_add_macvlan_element_data *add_list;
2437 struct i40e_aqc_remove_macvlan_element_data *del_list;
2439 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2440 usleep_range(1000, 2000);
2443 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2446 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2447 vsi->current_netdev_flags = vsi->netdev->flags;
2450 INIT_HLIST_HEAD(&tmp_add_list);
2451 INIT_HLIST_HEAD(&tmp_del_list);
2453 if (vsi->type == I40E_VSI_SRIOV)
2454 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2455 else if (vsi->type != I40E_VSI_MAIN)
2456 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2458 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2459 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2461 spin_lock_bh(&vsi->mac_filter_hash_lock);
2462 /* Create a list of filters to delete. */
2463 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2464 if (f->state == I40E_FILTER_REMOVE) {
2465 /* Move the element into temporary del_list */
2466 hash_del(&f->hlist);
2467 hlist_add_head(&f->hlist, &tmp_del_list);
2469 /* Avoid counting removed filters */
2472 if (f->state == I40E_FILTER_NEW) {
2473 /* Create a temporary i40e_new_mac_filter */
2474 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2476 goto err_no_memory_locked;
2478 /* Store pointer to the real filter */
2480 new->state = f->state;
2482 /* Add it to the hash list */
2483 hlist_add_head(&new->hlist, &tmp_add_list);
2486 /* Count the number of active (current and new) VLAN
2487 * filters we have now. Does not count filters which
2488 * are marked for deletion.
2494 retval = i40e_correct_mac_vlan_filters(vsi,
2499 hlist_for_each_entry(new, &tmp_add_list, hlist)
2500 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2503 goto err_no_memory_locked;
2505 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2508 /* Now process 'del_list' outside the lock */
2509 if (!hlist_empty(&tmp_del_list)) {
2510 filter_list_len = hw->aq.asq_buf_size /
2511 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2512 list_size = filter_list_len *
2513 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2514 del_list = kzalloc(list_size, GFP_ATOMIC);
2518 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2521 /* handle broadcast filters by updating the broadcast
2522 * promiscuous flag and release filter list.
2524 if (is_broadcast_ether_addr(f->macaddr)) {
2525 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2527 hlist_del(&f->hlist);
2532 /* add to delete list */
2533 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2534 if (f->vlan == I40E_VLAN_ANY) {
2535 del_list[num_del].vlan_tag = 0;
2536 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2538 del_list[num_del].vlan_tag =
2539 cpu_to_le16((u16)(f->vlan));
2542 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2543 del_list[num_del].flags = cmd_flags;
2546 /* flush a full buffer */
2547 if (num_del == filter_list_len) {
2548 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2550 memset(del_list, 0, list_size);
2553 /* Release memory for MAC filter entries which were
2554 * synced up with HW.
2556 hlist_del(&f->hlist);
2561 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2569 if (!hlist_empty(&tmp_add_list)) {
2570 /* Do all the adds now. */
2571 filter_list_len = hw->aq.asq_buf_size /
2572 sizeof(struct i40e_aqc_add_macvlan_element_data);
2573 list_size = filter_list_len *
2574 sizeof(struct i40e_aqc_add_macvlan_element_data);
2575 add_list = kzalloc(list_size, GFP_ATOMIC);
2580 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2581 /* handle broadcast filters by updating the broadcast
2582 * promiscuous flag instead of adding a MAC filter.
2584 if (is_broadcast_ether_addr(new->f->macaddr)) {
2585 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2587 new->state = I40E_FILTER_FAILED;
2589 new->state = I40E_FILTER_ACTIVE;
2593 /* add to add array */
2597 ether_addr_copy(add_list[num_add].mac_addr,
2599 if (new->f->vlan == I40E_VLAN_ANY) {
2600 add_list[num_add].vlan_tag = 0;
2601 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2603 add_list[num_add].vlan_tag =
2604 cpu_to_le16((u16)(new->f->vlan));
2606 add_list[num_add].queue_number = 0;
2607 /* set invalid match method for later detection */
2608 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2609 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2610 add_list[num_add].flags = cpu_to_le16(cmd_flags);
2613 /* flush a full buffer */
2614 if (num_add == filter_list_len) {
2615 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2617 memset(add_list, 0, list_size);
2622 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2625 /* Now move all of the filters from the temp add list back to
2628 spin_lock_bh(&vsi->mac_filter_hash_lock);
2629 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2630 /* Only update the state if we're still NEW */
2631 if (new->f->state == I40E_FILTER_NEW)
2632 new->f->state = new->state;
2633 hlist_del(&new->hlist);
2634 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2637 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2642 /* Determine the number of active and failed filters. */
2643 spin_lock_bh(&vsi->mac_filter_hash_lock);
2644 vsi->active_filters = 0;
2645 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2646 if (f->state == I40E_FILTER_ACTIVE)
2647 vsi->active_filters++;
2648 else if (f->state == I40E_FILTER_FAILED)
2651 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2653 /* Check if we are able to exit overflow promiscuous mode. We can
2654 * safely exit if we didn't just enter, we no longer have any failed
2655 * filters, and we have reduced filters below the threshold value.
2657 if (old_overflow && !failed_filters &&
2658 vsi->active_filters < vsi->promisc_threshold) {
2659 dev_info(&pf->pdev->dev,
2660 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2662 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2663 vsi->promisc_threshold = 0;
2666 /* if the VF is not trusted do not do promisc */
2667 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2668 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2672 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2674 /* If we are entering overflow promiscuous, we need to calculate a new
2675 * threshold for when we are safe to exit
2677 if (!old_overflow && new_overflow)
2678 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2680 /* check for changes in promiscuous modes */
2681 if (changed_flags & IFF_ALLMULTI) {
2682 bool cur_multipromisc;
2684 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2685 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2690 retval = i40e_aq_rc_to_posix(aq_ret,
2691 hw->aq.asq_last_status);
2692 dev_info(&pf->pdev->dev,
2693 "set multi promisc failed on %s, err %s aq_err %s\n",
2695 i40e_stat_str(hw, aq_ret),
2696 i40e_aq_str(hw, hw->aq.asq_last_status));
2698 dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2699 cur_multipromisc ? "entering" : "leaving");
2703 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2706 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2708 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2710 retval = i40e_aq_rc_to_posix(aq_ret,
2711 hw->aq.asq_last_status);
2712 dev_info(&pf->pdev->dev,
2713 "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2714 cur_promisc ? "on" : "off",
2716 i40e_stat_str(hw, aq_ret),
2717 i40e_aq_str(hw, hw->aq.asq_last_status));
2721 /* if something went wrong then set the changed flag so we try again */
2723 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2725 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2729 /* Restore elements on the temporary add and delete lists */
2730 spin_lock_bh(&vsi->mac_filter_hash_lock);
2731 err_no_memory_locked:
2732 i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2733 i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2734 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2736 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2737 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2742 * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2743 * @pf: board private structure
2745 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2751 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2753 if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2754 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2758 for (v = 0; v < pf->num_alloc_vsi; v++) {
2760 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2761 !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2762 int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2765 /* come back and try again later */
2766 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2775 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2778 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2780 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2781 return I40E_RXBUFFER_2048;
2783 return I40E_RXBUFFER_3072;
2787 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2788 * @netdev: network interface device structure
2789 * @new_mtu: new value for maximum frame size
2791 * Returns 0 on success, negative on failure
2793 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2795 struct i40e_netdev_priv *np = netdev_priv(netdev);
2796 struct i40e_vsi *vsi = np->vsi;
2797 struct i40e_pf *pf = vsi->back;
2799 if (i40e_enabled_xdp_vsi(vsi)) {
2800 int frame_size = new_mtu + I40E_PACKET_HDR_PAD;
2802 if (frame_size > i40e_max_xdp_frame_size(vsi))
2806 netdev_dbg(netdev, "changing MTU from %d to %d\n",
2807 netdev->mtu, new_mtu);
2808 netdev->mtu = new_mtu;
2809 if (netif_running(netdev))
2810 i40e_vsi_reinit_locked(vsi);
2811 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2812 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2817 * i40e_ioctl - Access the hwtstamp interface
2818 * @netdev: network interface device structure
2819 * @ifr: interface request data
2820 * @cmd: ioctl command
2822 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2824 struct i40e_netdev_priv *np = netdev_priv(netdev);
2825 struct i40e_pf *pf = np->vsi->back;
2829 return i40e_ptp_get_ts_config(pf, ifr);
2831 return i40e_ptp_set_ts_config(pf, ifr);
2838 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2839 * @vsi: the vsi being adjusted
2841 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2843 struct i40e_vsi_context ctxt;
2846 /* Don't modify stripping options if a port VLAN is active */
2850 if ((vsi->info.valid_sections &
2851 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2852 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2853 return; /* already enabled */
2855 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2856 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2857 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2859 ctxt.seid = vsi->seid;
2860 ctxt.info = vsi->info;
2861 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2863 dev_info(&vsi->back->pdev->dev,
2864 "update vlan stripping failed, err %s aq_err %s\n",
2865 i40e_stat_str(&vsi->back->hw, ret),
2866 i40e_aq_str(&vsi->back->hw,
2867 vsi->back->hw.aq.asq_last_status));
2872 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2873 * @vsi: the vsi being adjusted
2875 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2877 struct i40e_vsi_context ctxt;
2880 /* Don't modify stripping options if a port VLAN is active */
2884 if ((vsi->info.valid_sections &
2885 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2886 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2887 I40E_AQ_VSI_PVLAN_EMOD_MASK))
2888 return; /* already disabled */
2890 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2891 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2892 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2894 ctxt.seid = vsi->seid;
2895 ctxt.info = vsi->info;
2896 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2898 dev_info(&vsi->back->pdev->dev,
2899 "update vlan stripping failed, err %s aq_err %s\n",
2900 i40e_stat_str(&vsi->back->hw, ret),
2901 i40e_aq_str(&vsi->back->hw,
2902 vsi->back->hw.aq.asq_last_status));
2907 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2908 * @vsi: the vsi being configured
2909 * @vid: vlan id to be added (0 = untagged only , -1 = any)
2911 * This is a helper function for adding a new MAC/VLAN filter with the
2912 * specified VLAN for each existing MAC address already in the hash table.
2913 * This function does *not* perform any accounting to update filters based on
2916 * NOTE: this function expects to be called while under the
2917 * mac_filter_hash_lock
2919 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2921 struct i40e_mac_filter *f, *add_f;
2922 struct hlist_node *h;
2925 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2926 if (f->state == I40E_FILTER_REMOVE)
2928 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2930 dev_info(&vsi->back->pdev->dev,
2931 "Could not add vlan filter %d for %pM\n",
2941 * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2942 * @vsi: the VSI being configured
2943 * @vid: VLAN id to be added
2945 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2952 /* The network stack will attempt to add VID=0, with the intention to
2953 * receive priority tagged packets with a VLAN of 0. Our HW receives
2954 * these packets by default when configured to receive untagged
2955 * packets, so we don't need to add a filter for this case.
2956 * Additionally, HW interprets adding a VID=0 filter as meaning to
2957 * receive *only* tagged traffic and stops receiving untagged traffic.
2958 * Thus, we do not want to actually add a filter for VID=0
2963 /* Locked once because all functions invoked below iterates list*/
2964 spin_lock_bh(&vsi->mac_filter_hash_lock);
2965 err = i40e_add_vlan_all_mac(vsi, vid);
2966 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2970 /* schedule our worker thread which will take care of
2971 * applying the new filter changes
2973 i40e_service_event_schedule(vsi->back);
2978 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2979 * @vsi: the vsi being configured
2980 * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2982 * This function should be used to remove all VLAN filters which match the
2983 * given VID. It does not schedule the service event and does not take the
2984 * mac_filter_hash_lock so it may be combined with other operations under
2985 * a single invocation of the mac_filter_hash_lock.
2987 * NOTE: this function expects to be called while under the
2988 * mac_filter_hash_lock
2990 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2992 struct i40e_mac_filter *f;
2993 struct hlist_node *h;
2996 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2998 __i40e_del_filter(vsi, f);
3003 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3004 * @vsi: the VSI being configured
3005 * @vid: VLAN id to be removed
3007 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3009 if (!vid || vsi->info.pvid)
3012 spin_lock_bh(&vsi->mac_filter_hash_lock);
3013 i40e_rm_vlan_all_mac(vsi, vid);
3014 spin_unlock_bh(&vsi->mac_filter_hash_lock);
3016 /* schedule our worker thread which will take care of
3017 * applying the new filter changes
3019 i40e_service_event_schedule(vsi->back);
3023 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3024 * @netdev: network interface to be adjusted
3025 * @proto: unused protocol value
3026 * @vid: vlan id to be added
3028 * net_device_ops implementation for adding vlan ids
3030 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3031 __always_unused __be16 proto, u16 vid)
3033 struct i40e_netdev_priv *np = netdev_priv(netdev);
3034 struct i40e_vsi *vsi = np->vsi;
3037 if (vid >= VLAN_N_VID)
3040 ret = i40e_vsi_add_vlan(vsi, vid);
3042 set_bit(vid, vsi->active_vlans);
3048 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3049 * @netdev: network interface to be adjusted
3050 * @proto: unused protocol value
3051 * @vid: vlan id to be added
3053 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3054 __always_unused __be16 proto, u16 vid)
3056 struct i40e_netdev_priv *np = netdev_priv(netdev);
3057 struct i40e_vsi *vsi = np->vsi;
3059 if (vid >= VLAN_N_VID)
3061 set_bit(vid, vsi->active_vlans);
3065 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3066 * @netdev: network interface to be adjusted
3067 * @proto: unused protocol value
3068 * @vid: vlan id to be removed
3070 * net_device_ops implementation for removing vlan ids
3072 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3073 __always_unused __be16 proto, u16 vid)
3075 struct i40e_netdev_priv *np = netdev_priv(netdev);
3076 struct i40e_vsi *vsi = np->vsi;
3078 /* return code is ignored as there is nothing a user
3079 * can do about failure to remove and a log message was
3080 * already printed from the other function
3082 i40e_vsi_kill_vlan(vsi, vid);
3084 clear_bit(vid, vsi->active_vlans);
3090 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3091 * @vsi: the vsi being brought back up
3093 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3100 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3101 i40e_vlan_stripping_enable(vsi);
3103 i40e_vlan_stripping_disable(vsi);
3105 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3106 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3111 * i40e_vsi_add_pvid - Add pvid for the VSI
3112 * @vsi: the vsi being adjusted
3113 * @vid: the vlan id to set as a PVID
3115 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3117 struct i40e_vsi_context ctxt;
3120 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3121 vsi->info.pvid = cpu_to_le16(vid);
3122 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3123 I40E_AQ_VSI_PVLAN_INSERT_PVID |
3124 I40E_AQ_VSI_PVLAN_EMOD_STR;
3126 ctxt.seid = vsi->seid;
3127 ctxt.info = vsi->info;
3128 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3130 dev_info(&vsi->back->pdev->dev,
3131 "add pvid failed, err %s aq_err %s\n",
3132 i40e_stat_str(&vsi->back->hw, ret),
3133 i40e_aq_str(&vsi->back->hw,
3134 vsi->back->hw.aq.asq_last_status));
3142 * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3143 * @vsi: the vsi being adjusted
3145 * Just use the vlan_rx_register() service to put it back to normal
3147 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3151 i40e_vlan_stripping_disable(vsi);
3155 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3156 * @vsi: ptr to the VSI
3158 * If this function returns with an error, then it's possible one or
3159 * more of the rings is populated (while the rest are not). It is the
3160 * callers duty to clean those orphaned rings.
3162 * Return 0 on success, negative on failure
3164 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3168 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3169 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3171 if (!i40e_enabled_xdp_vsi(vsi))
3174 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3175 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3181 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3182 * @vsi: ptr to the VSI
3184 * Free VSI's transmit software resources
3186 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3190 if (vsi->tx_rings) {
3191 for (i = 0; i < vsi->num_queue_pairs; i++)
3192 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3193 i40e_free_tx_resources(vsi->tx_rings[i]);
3196 if (vsi->xdp_rings) {
3197 for (i = 0; i < vsi->num_queue_pairs; i++)
3198 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3199 i40e_free_tx_resources(vsi->xdp_rings[i]);
3204 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3205 * @vsi: ptr to the VSI
3207 * If this function returns with an error, then it's possible one or
3208 * more of the rings is populated (while the rest are not). It is the
3209 * callers duty to clean those orphaned rings.
3211 * Return 0 on success, negative on failure
3213 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3217 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3218 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3223 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3224 * @vsi: ptr to the VSI
3226 * Free all receive software resources
3228 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3235 for (i = 0; i < vsi->num_queue_pairs; i++)
3236 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3237 i40e_free_rx_resources(vsi->rx_rings[i]);
3241 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3242 * @ring: The Tx ring to configure
3244 * This enables/disables XPS for a given Tx descriptor ring
3245 * based on the TCs enabled for the VSI that ring belongs to.
3247 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3251 if (!ring->q_vector || !ring->netdev || ring->ch)
3254 /* We only initialize XPS once, so as not to overwrite user settings */
3255 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3258 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3259 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3264 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3265 * @ring: The Tx or Rx ring
3267 * Returns the AF_XDP buffer pool or NULL.
3269 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3271 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3272 int qid = ring->queue_index;
3274 if (ring_is_xdp(ring))
3275 qid -= ring->vsi->alloc_queue_pairs;
3277 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3280 return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3284 * i40e_configure_tx_ring - Configure a transmit ring context and rest
3285 * @ring: The Tx ring to configure
3287 * Configure the Tx descriptor ring in the HMC context.
3289 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3291 struct i40e_vsi *vsi = ring->vsi;
3292 u16 pf_q = vsi->base_queue + ring->queue_index;
3293 struct i40e_hw *hw = &vsi->back->hw;
3294 struct i40e_hmc_obj_txq tx_ctx;
3295 i40e_status err = 0;
3298 if (ring_is_xdp(ring))
3299 ring->xsk_pool = i40e_xsk_pool(ring);
3301 /* some ATR related tx ring init */
3302 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3303 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3304 ring->atr_count = 0;
3306 ring->atr_sample_rate = 0;
3310 i40e_config_xps_tx_ring(ring);
3312 /* clear the context structure first */
3313 memset(&tx_ctx, 0, sizeof(tx_ctx));
3315 tx_ctx.new_context = 1;
3316 tx_ctx.base = (ring->dma / 128);
3317 tx_ctx.qlen = ring->count;
3318 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3319 I40E_FLAG_FD_ATR_ENABLED));
3320 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3321 /* FDIR VSI tx ring can still use RS bit and writebacks */
3322 if (vsi->type != I40E_VSI_FDIR)
3323 tx_ctx.head_wb_ena = 1;
3324 tx_ctx.head_wb_addr = ring->dma +
3325 (ring->count * sizeof(struct i40e_tx_desc));
3327 /* As part of VSI creation/update, FW allocates certain
3328 * Tx arbitration queue sets for each TC enabled for
3329 * the VSI. The FW returns the handles to these queue
3330 * sets as part of the response buffer to Add VSI,
3331 * Update VSI, etc. AQ commands. It is expected that
3332 * these queue set handles be associated with the Tx
3333 * queues by the driver as part of the TX queue context
3334 * initialization. This has to be done regardless of
3335 * DCB as by default everything is mapped to TC0.
3340 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3343 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3345 tx_ctx.rdylist_act = 0;
3347 /* clear the context in the HMC */
3348 err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3350 dev_info(&vsi->back->pdev->dev,
3351 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3352 ring->queue_index, pf_q, err);
3356 /* set the context in the HMC */
3357 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3359 dev_info(&vsi->back->pdev->dev,
3360 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3361 ring->queue_index, pf_q, err);
3365 /* Now associate this queue with this PCI function */
3367 if (ring->ch->type == I40E_VSI_VMDQ2)
3368 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3372 qtx_ctl |= (ring->ch->vsi_number <<
3373 I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3374 I40E_QTX_CTL_VFVM_INDX_MASK;
3376 if (vsi->type == I40E_VSI_VMDQ2) {
3377 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3378 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3379 I40E_QTX_CTL_VFVM_INDX_MASK;
3381 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3385 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3386 I40E_QTX_CTL_PF_INDX_MASK);
3387 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3390 /* cache tail off for easier writes later */
3391 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3397 * i40e_configure_rx_ring - Configure a receive ring context
3398 * @ring: The Rx ring to configure
3400 * Configure the Rx descriptor ring in the HMC context.
3402 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3404 struct i40e_vsi *vsi = ring->vsi;
3405 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3406 u16 pf_q = vsi->base_queue + ring->queue_index;
3407 struct i40e_hw *hw = &vsi->back->hw;
3408 struct i40e_hmc_obj_rxq rx_ctx;
3409 i40e_status err = 0;
3413 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3415 /* clear the context structure first */
3416 memset(&rx_ctx, 0, sizeof(rx_ctx));
3418 if (ring->vsi->type == I40E_VSI_MAIN)
3419 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3421 ring->xsk_pool = i40e_xsk_pool(ring);
3422 if (ring->xsk_pool) {
3424 xsk_pool_get_rx_frame_size(ring->xsk_pool);
3425 /* For AF_XDP ZC, we disallow packets to span on
3426 * multiple buffers, thus letting us skip that
3427 * handling in the fast-path.
3430 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3431 MEM_TYPE_XSK_BUFF_POOL,
3435 dev_info(&vsi->back->pdev->dev,
3436 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3440 ring->rx_buf_len = vsi->rx_buf_len;
3441 if (ring->vsi->type == I40E_VSI_MAIN) {
3442 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3443 MEM_TYPE_PAGE_SHARED,
3450 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3451 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3453 rx_ctx.base = (ring->dma / 128);
3454 rx_ctx.qlen = ring->count;
3456 /* use 16 byte descriptors */
3459 /* descriptor type is always zero
3462 rx_ctx.hsplit_0 = 0;
3464 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3465 if (hw->revision_id == 0)
3466 rx_ctx.lrxqthresh = 0;
3468 rx_ctx.lrxqthresh = 1;
3469 rx_ctx.crcstrip = 1;
3471 /* this controls whether VLAN is stripped from inner headers */
3473 /* set the prefena field to 1 because the manual says to */
3476 /* clear the context in the HMC */
3477 err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3479 dev_info(&vsi->back->pdev->dev,
3480 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3481 ring->queue_index, pf_q, err);
3485 /* set the context in the HMC */
3486 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3488 dev_info(&vsi->back->pdev->dev,
3489 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3490 ring->queue_index, pf_q, err);
3494 /* configure Rx buffer alignment */
3495 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3496 clear_ring_build_skb_enabled(ring);
3498 set_ring_build_skb_enabled(ring);
3500 /* cache tail for quicker writes, and clear the reg before use */
3501 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3502 writel(0, ring->tail);
3504 if (ring->xsk_pool) {
3505 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3506 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3508 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3511 /* Log this in case the user has forgotten to give the kernel
3512 * any buffers, even later in the application.
3514 dev_info(&vsi->back->pdev->dev,
3515 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3516 ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3517 ring->queue_index, pf_q);
3524 * i40e_vsi_configure_tx - Configure the VSI for Tx
3525 * @vsi: VSI structure describing this set of rings and resources
3527 * Configure the Tx VSI for operation.
3529 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3534 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3535 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3537 if (err || !i40e_enabled_xdp_vsi(vsi))
3540 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3541 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3547 * i40e_vsi_configure_rx - Configure the VSI for Rx
3548 * @vsi: the VSI being configured
3550 * Configure the Rx VSI for operation.
3552 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3557 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3558 vsi->max_frame = I40E_MAX_RXBUFFER;
3559 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3560 #if (PAGE_SIZE < 8192)
3561 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3562 (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3563 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3564 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3567 vsi->max_frame = I40E_MAX_RXBUFFER;
3568 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3572 /* set up individual rings */
3573 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3574 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3580 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3581 * @vsi: ptr to the VSI
3583 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3585 struct i40e_ring *tx_ring, *rx_ring;
3586 u16 qoffset, qcount;
3589 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3590 /* Reset the TC information */
3591 for (i = 0; i < vsi->num_queue_pairs; i++) {
3592 rx_ring = vsi->rx_rings[i];
3593 tx_ring = vsi->tx_rings[i];
3594 rx_ring->dcb_tc = 0;
3595 tx_ring->dcb_tc = 0;
3600 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3601 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3604 qoffset = vsi->tc_config.tc_info[n].qoffset;
3605 qcount = vsi->tc_config.tc_info[n].qcount;
3606 for (i = qoffset; i < (qoffset + qcount); i++) {
3607 rx_ring = vsi->rx_rings[i];
3608 tx_ring = vsi->tx_rings[i];
3609 rx_ring->dcb_tc = n;
3610 tx_ring->dcb_tc = n;
3616 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3617 * @vsi: ptr to the VSI
3619 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3622 i40e_set_rx_mode(vsi->netdev);
3626 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3627 * @vsi: Pointer to the targeted VSI
3629 * This function replays the hlist on the hw where all the SB Flow Director
3630 * filters were saved.
3632 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3634 struct i40e_fdir_filter *filter;
3635 struct i40e_pf *pf = vsi->back;
3636 struct hlist_node *node;
3638 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3641 /* Reset FDir counters as we're replaying all existing filters */
3642 pf->fd_tcp4_filter_cnt = 0;
3643 pf->fd_udp4_filter_cnt = 0;
3644 pf->fd_sctp4_filter_cnt = 0;
3645 pf->fd_ip4_filter_cnt = 0;
3647 hlist_for_each_entry_safe(filter, node,
3648 &pf->fdir_filter_list, fdir_node) {
3649 i40e_add_del_fdir(vsi, filter, true);
3654 * i40e_vsi_configure - Set up the VSI for action
3655 * @vsi: the VSI being configured
3657 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3661 i40e_set_vsi_rx_mode(vsi);
3662 i40e_restore_vlan(vsi);
3663 i40e_vsi_config_dcb_rings(vsi);
3664 err = i40e_vsi_configure_tx(vsi);
3666 err = i40e_vsi_configure_rx(vsi);
3672 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3673 * @vsi: the VSI being configured
3675 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3677 bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3678 struct i40e_pf *pf = vsi->back;
3679 struct i40e_hw *hw = &pf->hw;
3684 /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3685 * and PFINT_LNKLSTn registers, e.g.:
3686 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
3688 qp = vsi->base_queue;
3689 vector = vsi->base_vector;
3690 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3691 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3693 q_vector->rx.next_update = jiffies + 1;
3694 q_vector->rx.target_itr =
3695 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3696 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3697 q_vector->rx.target_itr >> 1);
3698 q_vector->rx.current_itr = q_vector->rx.target_itr;
3700 q_vector->tx.next_update = jiffies + 1;
3701 q_vector->tx.target_itr =
3702 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3703 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3704 q_vector->tx.target_itr >> 1);
3705 q_vector->tx.current_itr = q_vector->tx.target_itr;
3707 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3708 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3710 /* Linked list for the queuepairs assigned to this vector */
3711 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3712 for (q = 0; q < q_vector->num_ringpairs; q++) {
3713 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3716 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3717 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3718 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3719 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3720 (I40E_QUEUE_TYPE_TX <<
3721 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3723 wr32(hw, I40E_QINT_RQCTL(qp), val);
3726 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3727 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3728 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3729 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3730 (I40E_QUEUE_TYPE_TX <<
3731 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3733 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3736 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3737 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3738 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3739 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3740 (I40E_QUEUE_TYPE_RX <<
3741 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3743 /* Terminate the linked list */
3744 if (q == (q_vector->num_ringpairs - 1))
3745 val |= (I40E_QUEUE_END_OF_LIST <<
3746 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3748 wr32(hw, I40E_QINT_TQCTL(qp), val);
3757 * i40e_enable_misc_int_causes - enable the non-queue interrupts
3758 * @pf: pointer to private device data structure
3760 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3762 struct i40e_hw *hw = &pf->hw;
3765 /* clear things first */
3766 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
3767 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
3769 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3770 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3771 I40E_PFINT_ICR0_ENA_GRST_MASK |
3772 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3773 I40E_PFINT_ICR0_ENA_GPIO_MASK |
3774 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3775 I40E_PFINT_ICR0_ENA_VFLR_MASK |
3776 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3778 if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3779 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3781 if (pf->flags & I40E_FLAG_PTP)
3782 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3784 wr32(hw, I40E_PFINT_ICR0_ENA, val);
3786 /* SW_ITR_IDX = 0, but don't change INTENA */
3787 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3788 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3790 /* OTHER_ITR_IDX = 0 */
3791 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3795 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3796 * @vsi: the VSI being configured
3798 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3800 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3801 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3802 struct i40e_pf *pf = vsi->back;
3803 struct i40e_hw *hw = &pf->hw;
3806 /* set the ITR configuration */
3807 q_vector->rx.next_update = jiffies + 1;
3808 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3809 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3810 q_vector->rx.current_itr = q_vector->rx.target_itr;
3811 q_vector->tx.next_update = jiffies + 1;
3812 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3813 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3814 q_vector->tx.current_itr = q_vector->tx.target_itr;
3816 i40e_enable_misc_int_causes(pf);
3818 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3819 wr32(hw, I40E_PFINT_LNKLST0, 0);
3821 /* Associate the queue pair to the vector and enable the queue int */
3822 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3823 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3824 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3825 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3827 wr32(hw, I40E_QINT_RQCTL(0), val);
3829 if (i40e_enabled_xdp_vsi(vsi)) {
3830 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3831 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3833 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3835 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3838 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3839 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3840 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3842 wr32(hw, I40E_QINT_TQCTL(0), val);
3847 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3848 * @pf: board private structure
3850 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3852 struct i40e_hw *hw = &pf->hw;
3854 wr32(hw, I40E_PFINT_DYN_CTL0,
3855 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3860 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3861 * @pf: board private structure
3863 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3865 struct i40e_hw *hw = &pf->hw;
3868 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3869 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3870 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3872 wr32(hw, I40E_PFINT_DYN_CTL0, val);
3877 * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3878 * @irq: interrupt number
3879 * @data: pointer to a q_vector
3881 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3883 struct i40e_q_vector *q_vector = data;
3885 if (!q_vector->tx.ring && !q_vector->rx.ring)
3888 napi_schedule_irqoff(&q_vector->napi);
3894 * i40e_irq_affinity_notify - Callback for affinity changes
3895 * @notify: context as to what irq was changed
3896 * @mask: the new affinity mask
3898 * This is a callback function used by the irq_set_affinity_notifier function
3899 * so that we may register to receive changes to the irq affinity masks.
3901 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3902 const cpumask_t *mask)
3904 struct i40e_q_vector *q_vector =
3905 container_of(notify, struct i40e_q_vector, affinity_notify);
3907 cpumask_copy(&q_vector->affinity_mask, mask);
3911 * i40e_irq_affinity_release - Callback for affinity notifier release
3912 * @ref: internal core kernel usage
3914 * This is a callback function used by the irq_set_affinity_notifier function
3915 * to inform the current notification subscriber that they will no longer
3916 * receive notifications.
3918 static void i40e_irq_affinity_release(struct kref *ref) {}
3921 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3922 * @vsi: the VSI being configured
3923 * @basename: name for the vector
3925 * Allocates MSI-X vectors and requests interrupts from the kernel.
3927 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3929 int q_vectors = vsi->num_q_vectors;
3930 struct i40e_pf *pf = vsi->back;
3931 int base = vsi->base_vector;
3938 for (vector = 0; vector < q_vectors; vector++) {
3939 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3941 irq_num = pf->msix_entries[base + vector].vector;
3943 if (q_vector->tx.ring && q_vector->rx.ring) {
3944 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3945 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3947 } else if (q_vector->rx.ring) {
3948 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3949 "%s-%s-%d", basename, "rx", rx_int_idx++);
3950 } else if (q_vector->tx.ring) {
3951 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3952 "%s-%s-%d", basename, "tx", tx_int_idx++);
3954 /* skip this unused q_vector */
3957 err = request_irq(irq_num,
3963 dev_info(&pf->pdev->dev,
3964 "MSIX request_irq failed, error: %d\n", err);
3965 goto free_queue_irqs;
3968 /* register for affinity change notifications */
3969 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3970 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3971 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3972 /* Spread affinity hints out across online CPUs.
3974 * get_cpu_mask returns a static constant mask with
3975 * a permanent lifetime so it's ok to pass to
3976 * irq_set_affinity_hint without making a copy.
3978 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3979 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3982 vsi->irqs_ready = true;
3988 irq_num = pf->msix_entries[base + vector].vector;
3989 irq_set_affinity_notifier(irq_num, NULL);
3990 irq_set_affinity_hint(irq_num, NULL);
3991 free_irq(irq_num, &vsi->q_vectors[vector]);
3997 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3998 * @vsi: the VSI being un-configured
4000 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4002 struct i40e_pf *pf = vsi->back;
4003 struct i40e_hw *hw = &pf->hw;
4004 int base = vsi->base_vector;
4007 /* disable interrupt causation from each queue */
4008 for (i = 0; i < vsi->num_queue_pairs; i++) {
4011 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4012 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4013 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4015 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4016 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4017 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4019 if (!i40e_enabled_xdp_vsi(vsi))
4021 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4024 /* disable each interrupt */
4025 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4026 for (i = vsi->base_vector;
4027 i < (vsi->num_q_vectors + vsi->base_vector); i++)
4028 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4031 for (i = 0; i < vsi->num_q_vectors; i++)
4032 synchronize_irq(pf->msix_entries[i + base].vector);
4034 /* Legacy and MSI mode - this stops all interrupt handling */
4035 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4036 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4038 synchronize_irq(pf->pdev->irq);
4043 * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4044 * @vsi: the VSI being configured
4046 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4048 struct i40e_pf *pf = vsi->back;
4051 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4052 for (i = 0; i < vsi->num_q_vectors; i++)
4053 i40e_irq_dynamic_enable(vsi, i);
4055 i40e_irq_dynamic_enable_icr0(pf);
4058 i40e_flush(&pf->hw);
4063 * i40e_free_misc_vector - Free the vector that handles non-queue events
4064 * @pf: board private structure
4066 static void i40e_free_misc_vector(struct i40e_pf *pf)
4069 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4070 i40e_flush(&pf->hw);
4072 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4073 synchronize_irq(pf->msix_entries[0].vector);
4074 free_irq(pf->msix_entries[0].vector, pf);
4075 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4080 * i40e_intr - MSI/Legacy and non-queue interrupt handler
4081 * @irq: interrupt number
4082 * @data: pointer to a q_vector
4084 * This is the handler used for all MSI/Legacy interrupts, and deals
4085 * with both queue and non-queue interrupts. This is also used in
4086 * MSIX mode to handle the non-queue interrupts.
4088 static irqreturn_t i40e_intr(int irq, void *data)
4090 struct i40e_pf *pf = (struct i40e_pf *)data;
4091 struct i40e_hw *hw = &pf->hw;
4092 irqreturn_t ret = IRQ_NONE;
4093 u32 icr0, icr0_remaining;
4096 icr0 = rd32(hw, I40E_PFINT_ICR0);
4097 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4099 /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4100 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4103 /* if interrupt but no bits showing, must be SWINT */
4104 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4105 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4108 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
4109 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4110 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4111 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4112 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4115 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4116 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4117 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4118 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4120 /* We do not have a way to disarm Queue causes while leaving
4121 * interrupt enabled for all other causes, ideally
4122 * interrupt should be disabled while we are in NAPI but
4123 * this is not a performance path and napi_schedule()
4124 * can deal with rescheduling.
4126 if (!test_bit(__I40E_DOWN, pf->state))
4127 napi_schedule_irqoff(&q_vector->napi);
4130 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4131 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4132 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4133 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4136 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4137 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4138 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4141 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4142 /* disable any further VFLR event notifications */
4143 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4144 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4146 reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4147 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4149 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4150 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4154 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4155 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4156 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4157 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4158 val = rd32(hw, I40E_GLGEN_RSTAT);
4159 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4160 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4161 if (val == I40E_RESET_CORER) {
4163 } else if (val == I40E_RESET_GLOBR) {
4165 } else if (val == I40E_RESET_EMPR) {
4167 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4171 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4172 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4173 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4174 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4175 rd32(hw, I40E_PFHMC_ERRORINFO),
4176 rd32(hw, I40E_PFHMC_ERRORDATA));
4179 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4180 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4182 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
4183 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4184 i40e_ptp_tx_hwtstamp(pf);
4188 /* If a critical error is pending we have no choice but to reset the
4190 * Report and mask out any remaining unexpected interrupts.
4192 icr0_remaining = icr0 & ena_mask;
4193 if (icr0_remaining) {
4194 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4196 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4197 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4198 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4199 dev_info(&pf->pdev->dev, "device will be reset\n");
4200 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4201 i40e_service_event_schedule(pf);
4203 ena_mask &= ~icr0_remaining;
4208 /* re-enable interrupt causes */
4209 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4210 if (!test_bit(__I40E_DOWN, pf->state) ||
4211 test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4212 i40e_service_event_schedule(pf);
4213 i40e_irq_dynamic_enable_icr0(pf);
4220 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4221 * @tx_ring: tx ring to clean
4222 * @budget: how many cleans we're allowed
4224 * Returns true if there's any budget left (e.g. the clean is finished)
4226 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4228 struct i40e_vsi *vsi = tx_ring->vsi;
4229 u16 i = tx_ring->next_to_clean;
4230 struct i40e_tx_buffer *tx_buf;
4231 struct i40e_tx_desc *tx_desc;
4233 tx_buf = &tx_ring->tx_bi[i];
4234 tx_desc = I40E_TX_DESC(tx_ring, i);
4235 i -= tx_ring->count;
4238 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4240 /* if next_to_watch is not set then there is no work pending */
4244 /* prevent any other reads prior to eop_desc */
4247 /* if the descriptor isn't done, no work yet to do */
4248 if (!(eop_desc->cmd_type_offset_bsz &
4249 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4252 /* clear next_to_watch to prevent false hangs */
4253 tx_buf->next_to_watch = NULL;
4255 tx_desc->buffer_addr = 0;
4256 tx_desc->cmd_type_offset_bsz = 0;
4257 /* move past filter desc */
4262 i -= tx_ring->count;
4263 tx_buf = tx_ring->tx_bi;
4264 tx_desc = I40E_TX_DESC(tx_ring, 0);
4266 /* unmap skb header data */
4267 dma_unmap_single(tx_ring->dev,
4268 dma_unmap_addr(tx_buf, dma),
4269 dma_unmap_len(tx_buf, len),
4271 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4272 kfree(tx_buf->raw_buf);
4274 tx_buf->raw_buf = NULL;
4275 tx_buf->tx_flags = 0;
4276 tx_buf->next_to_watch = NULL;
4277 dma_unmap_len_set(tx_buf, len, 0);
4278 tx_desc->buffer_addr = 0;
4279 tx_desc->cmd_type_offset_bsz = 0;
4281 /* move us past the eop_desc for start of next FD desc */
4286 i -= tx_ring->count;
4287 tx_buf = tx_ring->tx_bi;
4288 tx_desc = I40E_TX_DESC(tx_ring, 0);
4291 /* update budget accounting */
4293 } while (likely(budget));
4295 i += tx_ring->count;
4296 tx_ring->next_to_clean = i;
4298 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4299 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4305 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4306 * @irq: interrupt number
4307 * @data: pointer to a q_vector
4309 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4311 struct i40e_q_vector *q_vector = data;
4312 struct i40e_vsi *vsi;
4314 if (!q_vector->tx.ring)
4317 vsi = q_vector->tx.ring->vsi;
4318 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4324 * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4325 * @vsi: the VSI being configured
4326 * @v_idx: vector index
4327 * @qp_idx: queue pair index
4329 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4331 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4332 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4333 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4335 tx_ring->q_vector = q_vector;
4336 tx_ring->next = q_vector->tx.ring;
4337 q_vector->tx.ring = tx_ring;
4338 q_vector->tx.count++;
4340 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4341 if (i40e_enabled_xdp_vsi(vsi)) {
4342 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4344 xdp_ring->q_vector = q_vector;
4345 xdp_ring->next = q_vector->tx.ring;
4346 q_vector->tx.ring = xdp_ring;
4347 q_vector->tx.count++;
4350 rx_ring->q_vector = q_vector;
4351 rx_ring->next = q_vector->rx.ring;
4352 q_vector->rx.ring = rx_ring;
4353 q_vector->rx.count++;
4357 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4358 * @vsi: the VSI being configured
4360 * This function maps descriptor rings to the queue-specific vectors
4361 * we were allotted through the MSI-X enabling code. Ideally, we'd have
4362 * one vector per queue pair, but on a constrained vector budget, we
4363 * group the queue pairs as "efficiently" as possible.
4365 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4367 int qp_remaining = vsi->num_queue_pairs;
4368 int q_vectors = vsi->num_q_vectors;
4373 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4374 * group them so there are multiple queues per vector.
4375 * It is also important to go through all the vectors available to be
4376 * sure that if we don't use all the vectors, that the remaining vectors
4377 * are cleared. This is especially important when decreasing the
4378 * number of queues in use.
4380 for (; v_start < q_vectors; v_start++) {
4381 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4383 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4385 q_vector->num_ringpairs = num_ringpairs;
4386 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4388 q_vector->rx.count = 0;
4389 q_vector->tx.count = 0;
4390 q_vector->rx.ring = NULL;
4391 q_vector->tx.ring = NULL;
4393 while (num_ringpairs--) {
4394 i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4402 * i40e_vsi_request_irq - Request IRQ from the OS
4403 * @vsi: the VSI being configured
4404 * @basename: name for the vector
4406 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4408 struct i40e_pf *pf = vsi->back;
4411 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4412 err = i40e_vsi_request_irq_msix(vsi, basename);
4413 else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4414 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4417 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4421 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4426 #ifdef CONFIG_NET_POLL_CONTROLLER
4428 * i40e_netpoll - A Polling 'interrupt' handler
4429 * @netdev: network interface device structure
4431 * This is used by netconsole to send skbs without having to re-enable
4432 * interrupts. It's not called while the normal interrupt routine is executing.
4434 static void i40e_netpoll(struct net_device *netdev)
4436 struct i40e_netdev_priv *np = netdev_priv(netdev);
4437 struct i40e_vsi *vsi = np->vsi;
4438 struct i40e_pf *pf = vsi->back;
4441 /* if interface is down do nothing */
4442 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4445 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4446 for (i = 0; i < vsi->num_q_vectors; i++)
4447 i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4449 i40e_intr(pf->pdev->irq, netdev);
4454 #define I40E_QTX_ENA_WAIT_COUNT 50
4457 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4458 * @pf: the PF being configured
4459 * @pf_q: the PF queue
4460 * @enable: enable or disable state of the queue
4462 * This routine will wait for the given Tx queue of the PF to reach the
4463 * enabled or disabled state.
4464 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4465 * multiple retries; else will return 0 in case of success.
4467 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4472 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4473 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4474 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4477 usleep_range(10, 20);
4479 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4486 * i40e_control_tx_q - Start or stop a particular Tx queue
4487 * @pf: the PF structure
4488 * @pf_q: the PF queue to configure
4489 * @enable: start or stop the queue
4491 * This function enables or disables a single queue. Note that any delay
4492 * required after the operation is expected to be handled by the caller of
4495 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4497 struct i40e_hw *hw = &pf->hw;
4501 /* warn the TX unit of coming changes */
4502 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4504 usleep_range(10, 20);
4506 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4507 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4508 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4509 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4511 usleep_range(1000, 2000);
4514 /* Skip if the queue is already in the requested state */
4515 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4518 /* turn on/off the queue */
4520 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4521 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4523 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4526 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4530 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4532 * @pf: the PF structure
4533 * @pf_q: the PF queue to configure
4534 * @is_xdp: true if the queue is used for XDP
4535 * @enable: start or stop the queue
4537 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4538 bool is_xdp, bool enable)
4542 i40e_control_tx_q(pf, pf_q, enable);
4544 /* wait for the change to finish */
4545 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4547 dev_info(&pf->pdev->dev,
4548 "VSI seid %d %sTx ring %d %sable timeout\n",
4549 seid, (is_xdp ? "XDP " : ""), pf_q,
4550 (enable ? "en" : "dis"));
4557 * i40e_vsi_enable_tx - Start a VSI's rings
4558 * @vsi: the VSI being configured
4560 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4562 struct i40e_pf *pf = vsi->back;
4563 int i, pf_q, ret = 0;
4565 pf_q = vsi->base_queue;
4566 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4567 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4569 false /*is xdp*/, true);
4573 if (!i40e_enabled_xdp_vsi(vsi))
4576 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4577 pf_q + vsi->alloc_queue_pairs,
4578 true /*is xdp*/, true);
4586 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4587 * @pf: the PF being configured
4588 * @pf_q: the PF queue
4589 * @enable: enable or disable state of the queue
4591 * This routine will wait for the given Rx queue of the PF to reach the
4592 * enabled or disabled state.
4593 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4594 * multiple retries; else will return 0 in case of success.
4596 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4601 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4602 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4603 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4606 usleep_range(10, 20);
4608 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4615 * i40e_control_rx_q - Start or stop a particular Rx queue
4616 * @pf: the PF structure
4617 * @pf_q: the PF queue to configure
4618 * @enable: start or stop the queue
4620 * This function enables or disables a single queue. Note that
4621 * any delay required after the operation is expected to be
4622 * handled by the caller of this function.
4624 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4626 struct i40e_hw *hw = &pf->hw;
4630 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4631 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4632 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4633 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4635 usleep_range(1000, 2000);
4638 /* Skip if the queue is already in the requested state */
4639 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4642 /* turn on/off the queue */
4644 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4646 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4648 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4652 * i40e_control_wait_rx_q
4653 * @pf: the PF structure
4654 * @pf_q: queue being configured
4655 * @enable: start or stop the rings
4657 * This function enables or disables a single queue along with waiting
4658 * for the change to finish. The caller of this function should handle
4659 * the delays needed in the case of disabling queues.
4661 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4665 i40e_control_rx_q(pf, pf_q, enable);
4667 /* wait for the change to finish */
4668 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4676 * i40e_vsi_enable_rx - Start a VSI's rings
4677 * @vsi: the VSI being configured
4679 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4681 struct i40e_pf *pf = vsi->back;
4682 int i, pf_q, ret = 0;
4684 pf_q = vsi->base_queue;
4685 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4686 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4688 dev_info(&pf->pdev->dev,
4689 "VSI seid %d Rx ring %d enable timeout\n",
4699 * i40e_vsi_start_rings - Start a VSI's rings
4700 * @vsi: the VSI being configured
4702 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4706 /* do rx first for enable and last for disable */
4707 ret = i40e_vsi_enable_rx(vsi);
4710 ret = i40e_vsi_enable_tx(vsi);
4715 #define I40E_DISABLE_TX_GAP_MSEC 50
4718 * i40e_vsi_stop_rings - Stop a VSI's rings
4719 * @vsi: the VSI being configured
4721 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4723 struct i40e_pf *pf = vsi->back;
4724 int pf_q, err, q_end;
4726 /* When port TX is suspended, don't wait */
4727 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4728 return i40e_vsi_stop_rings_no_wait(vsi);
4730 q_end = vsi->base_queue + vsi->num_queue_pairs;
4731 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4732 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4734 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4735 err = i40e_control_wait_rx_q(pf, pf_q, false);
4737 dev_info(&pf->pdev->dev,
4738 "VSI seid %d Rx ring %d dissable timeout\n",
4742 msleep(I40E_DISABLE_TX_GAP_MSEC);
4743 pf_q = vsi->base_queue;
4744 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4745 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4747 i40e_vsi_wait_queues_disabled(vsi);
4751 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4752 * @vsi: the VSI being shutdown
4754 * This function stops all the rings for a VSI but does not delay to verify
4755 * that rings have been disabled. It is expected that the caller is shutting
4756 * down multiple VSIs at once and will delay together for all the VSIs after
4757 * initiating the shutdown. This is particularly useful for shutting down lots
4758 * of VFs together. Otherwise, a large delay can be incurred while configuring
4759 * each VSI in serial.
4761 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4763 struct i40e_pf *pf = vsi->back;
4766 pf_q = vsi->base_queue;
4767 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4768 i40e_control_tx_q(pf, pf_q, false);
4769 i40e_control_rx_q(pf, pf_q, false);
4774 * i40e_vsi_free_irq - Free the irq association with the OS
4775 * @vsi: the VSI being configured
4777 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4779 struct i40e_pf *pf = vsi->back;
4780 struct i40e_hw *hw = &pf->hw;
4781 int base = vsi->base_vector;
4785 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4786 if (!vsi->q_vectors)
4789 if (!vsi->irqs_ready)
4792 vsi->irqs_ready = false;
4793 for (i = 0; i < vsi->num_q_vectors; i++) {
4798 irq_num = pf->msix_entries[vector].vector;
4800 /* free only the irqs that were actually requested */
4801 if (!vsi->q_vectors[i] ||
4802 !vsi->q_vectors[i]->num_ringpairs)
4805 /* clear the affinity notifier in the IRQ descriptor */
4806 irq_set_affinity_notifier(irq_num, NULL);
4807 /* remove our suggested affinity mask for this IRQ */
4808 irq_set_affinity_hint(irq_num, NULL);
4809 synchronize_irq(irq_num);
4810 free_irq(irq_num, vsi->q_vectors[i]);
4812 /* Tear down the interrupt queue link list
4814 * We know that they come in pairs and always
4815 * the Rx first, then the Tx. To clear the
4816 * link list, stick the EOL value into the
4817 * next_q field of the registers.
4819 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4820 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4821 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4822 val |= I40E_QUEUE_END_OF_LIST
4823 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4824 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4826 while (qp != I40E_QUEUE_END_OF_LIST) {
4829 val = rd32(hw, I40E_QINT_RQCTL(qp));
4831 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4832 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4833 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4834 I40E_QINT_RQCTL_INTEVENT_MASK);
4836 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4837 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4839 wr32(hw, I40E_QINT_RQCTL(qp), val);
4841 val = rd32(hw, I40E_QINT_TQCTL(qp));
4843 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4844 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4846 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4847 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4848 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4849 I40E_QINT_TQCTL_INTEVENT_MASK);
4851 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4852 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4854 wr32(hw, I40E_QINT_TQCTL(qp), val);
4859 free_irq(pf->pdev->irq, pf);
4861 val = rd32(hw, I40E_PFINT_LNKLST0);
4862 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4863 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4864 val |= I40E_QUEUE_END_OF_LIST
4865 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4866 wr32(hw, I40E_PFINT_LNKLST0, val);
4868 val = rd32(hw, I40E_QINT_RQCTL(qp));
4869 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4870 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4871 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4872 I40E_QINT_RQCTL_INTEVENT_MASK);
4874 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4875 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4877 wr32(hw, I40E_QINT_RQCTL(qp), val);
4879 val = rd32(hw, I40E_QINT_TQCTL(qp));
4881 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4882 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4883 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4884 I40E_QINT_TQCTL_INTEVENT_MASK);
4886 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4887 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4889 wr32(hw, I40E_QINT_TQCTL(qp), val);
4894 * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4895 * @vsi: the VSI being configured
4896 * @v_idx: Index of vector to be freed
4898 * This function frees the memory allocated to the q_vector. In addition if
4899 * NAPI is enabled it will delete any references to the NAPI struct prior
4900 * to freeing the q_vector.
4902 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4904 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4905 struct i40e_ring *ring;
4910 /* disassociate q_vector from rings */
4911 i40e_for_each_ring(ring, q_vector->tx)
4912 ring->q_vector = NULL;
4914 i40e_for_each_ring(ring, q_vector->rx)
4915 ring->q_vector = NULL;
4917 /* only VSI w/ an associated netdev is set up w/ NAPI */
4919 netif_napi_del(&q_vector->napi);
4921 vsi->q_vectors[v_idx] = NULL;
4923 kfree_rcu(q_vector, rcu);
4927 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4928 * @vsi: the VSI being un-configured
4930 * This frees the memory allocated to the q_vectors and
4931 * deletes references to the NAPI struct.
4933 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4937 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4938 i40e_free_q_vector(vsi, v_idx);
4942 * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4943 * @pf: board private structure
4945 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4947 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4948 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4949 pci_disable_msix(pf->pdev);
4950 kfree(pf->msix_entries);
4951 pf->msix_entries = NULL;
4952 kfree(pf->irq_pile);
4953 pf->irq_pile = NULL;
4954 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4955 pci_disable_msi(pf->pdev);
4957 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4961 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4962 * @pf: board private structure
4964 * We go through and clear interrupt specific resources and reset the structure
4965 * to pre-load conditions
4967 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4971 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
4972 i40e_free_misc_vector(pf);
4974 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4975 I40E_IWARP_IRQ_PILE_ID);
4977 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4978 for (i = 0; i < pf->num_alloc_vsi; i++)
4980 i40e_vsi_free_q_vectors(pf->vsi[i]);
4981 i40e_reset_interrupt_capability(pf);
4985 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4986 * @vsi: the VSI being configured
4988 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4995 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4996 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4998 if (q_vector->rx.ring || q_vector->tx.ring)
4999 napi_enable(&q_vector->napi);
5004 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5005 * @vsi: the VSI being configured
5007 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5014 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5015 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5017 if (q_vector->rx.ring || q_vector->tx.ring)
5018 napi_disable(&q_vector->napi);
5023 * i40e_vsi_close - Shut down a VSI
5024 * @vsi: the vsi to be quelled
5026 static void i40e_vsi_close(struct i40e_vsi *vsi)
5028 struct i40e_pf *pf = vsi->back;
5029 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5031 i40e_vsi_free_irq(vsi);
5032 i40e_vsi_free_tx_resources(vsi);
5033 i40e_vsi_free_rx_resources(vsi);
5034 vsi->current_netdev_flags = 0;
5035 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5036 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5037 set_bit(__I40E_CLIENT_RESET, pf->state);
5041 * i40e_quiesce_vsi - Pause a given VSI
5042 * @vsi: the VSI being paused
5044 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5046 if (test_bit(__I40E_VSI_DOWN, vsi->state))
5049 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5050 if (vsi->netdev && netif_running(vsi->netdev))
5051 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5053 i40e_vsi_close(vsi);
5057 * i40e_unquiesce_vsi - Resume a given VSI
5058 * @vsi: the VSI being resumed
5060 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5062 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5065 if (vsi->netdev && netif_running(vsi->netdev))
5066 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5068 i40e_vsi_open(vsi); /* this clears the DOWN bit */
5072 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5075 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5079 for (v = 0; v < pf->num_alloc_vsi; v++) {
5081 i40e_quiesce_vsi(pf->vsi[v]);
5086 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5089 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5093 for (v = 0; v < pf->num_alloc_vsi; v++) {
5095 i40e_unquiesce_vsi(pf->vsi[v]);
5100 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5101 * @vsi: the VSI being configured
5103 * Wait until all queues on a given VSI have been disabled.
5105 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5107 struct i40e_pf *pf = vsi->back;
5110 pf_q = vsi->base_queue;
5111 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5112 /* Check and wait for the Tx queue */
5113 ret = i40e_pf_txq_wait(pf, pf_q, false);
5115 dev_info(&pf->pdev->dev,
5116 "VSI seid %d Tx ring %d disable timeout\n",
5121 if (!i40e_enabled_xdp_vsi(vsi))
5124 /* Check and wait for the XDP Tx queue */
5125 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5128 dev_info(&pf->pdev->dev,
5129 "VSI seid %d XDP Tx ring %d disable timeout\n",
5134 /* Check and wait for the Rx queue */
5135 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5137 dev_info(&pf->pdev->dev,
5138 "VSI seid %d Rx ring %d disable timeout\n",
5147 #ifdef CONFIG_I40E_DCB
5149 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5152 * This function waits for the queues to be in disabled state for all the
5153 * VSIs that are managed by this PF.
5155 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5159 for (v = 0; v < pf->num_alloc_vsi; v++) {
5161 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5173 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5174 * @pf: pointer to PF
5176 * Get TC map for ISCSI PF type that will include iSCSI TC
5179 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5181 struct i40e_dcb_app_priority_table app;
5182 struct i40e_hw *hw = &pf->hw;
5183 u8 enabled_tc = 1; /* TC0 is always enabled */
5185 /* Get the iSCSI APP TLV */
5186 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5188 for (i = 0; i < dcbcfg->numapps; i++) {
5189 app = dcbcfg->app[i];
5190 if (app.selector == I40E_APP_SEL_TCPIP &&
5191 app.protocolid == I40E_APP_PROTOID_ISCSI) {
5192 tc = dcbcfg->etscfg.prioritytable[app.priority];
5193 enabled_tc |= BIT(tc);
5202 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
5203 * @dcbcfg: the corresponding DCBx configuration structure
5205 * Return the number of TCs from given DCBx configuration
5207 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5209 int i, tc_unused = 0;
5213 /* Scan the ETS Config Priority Table to find
5214 * traffic class enabled for a given priority
5215 * and create a bitmask of enabled TCs
5217 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5218 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5220 /* Now scan the bitmask to check for
5221 * contiguous TCs starting with TC0
5223 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5224 if (num_tc & BIT(i)) {
5228 pr_err("Non-contiguous TC - Disabling DCB\n");
5236 /* There is always at least TC0 */
5244 * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5245 * @dcbcfg: the corresponding DCBx configuration structure
5247 * Query the current DCB configuration and return the number of
5248 * traffic classes enabled from the given DCBX config
5250 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5252 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5256 for (i = 0; i < num_tc; i++)
5257 enabled_tc |= BIT(i);
5263 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5264 * @pf: PF being queried
5266 * Query the current MQPRIO configuration and return the number of
5267 * traffic classes enabled.
5269 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5271 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5272 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5273 u8 enabled_tc = 1, i;
5275 for (i = 1; i < num_tc; i++)
5276 enabled_tc |= BIT(i);
5281 * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5282 * @pf: PF being queried
5284 * Return number of traffic classes enabled for the given PF
5286 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5288 struct i40e_hw *hw = &pf->hw;
5289 u8 i, enabled_tc = 1;
5291 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5293 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5294 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5296 /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5297 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5300 /* SFP mode will be enabled for all TCs on port */
5301 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5302 return i40e_dcb_get_num_tc(dcbcfg);
5304 /* MFP mode return count of enabled TCs for this PF */
5305 if (pf->hw.func_caps.iscsi)
5306 enabled_tc = i40e_get_iscsi_tc_map(pf);
5308 return 1; /* Only TC0 */
5310 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5311 if (enabled_tc & BIT(i))
5318 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
5319 * @pf: PF being queried
5321 * Return a bitmap for enabled traffic classes for this PF.
5323 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5325 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5326 return i40e_mqprio_get_enabled_tc(pf);
5328 /* If neither MQPRIO nor DCB is enabled for this PF then just return
5331 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5332 return I40E_DEFAULT_TRAFFIC_CLASS;
5334 /* SFP mode we want PF to be enabled for all TCs */
5335 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5336 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5338 /* MFP enabled and iSCSI PF type */
5339 if (pf->hw.func_caps.iscsi)
5340 return i40e_get_iscsi_tc_map(pf);
5342 return I40E_DEFAULT_TRAFFIC_CLASS;
5346 * i40e_vsi_get_bw_info - Query VSI BW Information
5347 * @vsi: the VSI being queried
5349 * Returns 0 on success, negative value on failure
5351 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5353 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5354 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5355 struct i40e_pf *pf = vsi->back;
5356 struct i40e_hw *hw = &pf->hw;
5361 /* Get the VSI level BW configuration */
5362 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5364 dev_info(&pf->pdev->dev,
5365 "couldn't get PF vsi bw config, err %s aq_err %s\n",
5366 i40e_stat_str(&pf->hw, ret),
5367 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5371 /* Get the VSI level BW configuration per TC */
5372 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5375 dev_info(&pf->pdev->dev,
5376 "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5377 i40e_stat_str(&pf->hw, ret),
5378 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5382 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5383 dev_info(&pf->pdev->dev,
5384 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5385 bw_config.tc_valid_bits,
5386 bw_ets_config.tc_valid_bits);
5387 /* Still continuing */
5390 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5391 vsi->bw_max_quanta = bw_config.max_bw;
5392 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5393 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5394 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5395 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5396 vsi->bw_ets_limit_credits[i] =
5397 le16_to_cpu(bw_ets_config.credits[i]);
5398 /* 3 bits out of 4 for each TC */
5399 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5406 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5407 * @vsi: the VSI being configured
5408 * @enabled_tc: TC bitmap
5409 * @bw_share: BW shared credits per TC
5411 * Returns 0 on success, negative value on failure
5413 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5416 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5417 struct i40e_pf *pf = vsi->back;
5421 /* There is no need to reset BW when mqprio mode is on. */
5422 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5424 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5425 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5427 dev_info(&pf->pdev->dev,
5428 "Failed to reset tx rate for vsi->seid %u\n",
5432 bw_data.tc_valid_bits = enabled_tc;
5433 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5434 bw_data.tc_bw_credits[i] = bw_share[i];
5436 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5438 dev_info(&pf->pdev->dev,
5439 "AQ command Config VSI BW allocation per TC failed = %d\n",
5440 pf->hw.aq.asq_last_status);
5444 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5445 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5451 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5452 * @vsi: the VSI being configured
5453 * @enabled_tc: TC map to be enabled
5456 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5458 struct net_device *netdev = vsi->netdev;
5459 struct i40e_pf *pf = vsi->back;
5460 struct i40e_hw *hw = &pf->hw;
5463 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5469 netdev_reset_tc(netdev);
5473 /* Set up actual enabled TCs on the VSI */
5474 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5477 /* set per TC queues for the VSI */
5478 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5479 /* Only set TC queues for enabled tcs
5481 * e.g. For a VSI that has TC0 and TC3 enabled the
5482 * enabled_tc bitmap would be 0x00001001; the driver
5483 * will set the numtc for netdev as 2 that will be
5484 * referenced by the netdev layer as TC 0 and 1.
5486 if (vsi->tc_config.enabled_tc & BIT(i))
5487 netdev_set_tc_queue(netdev,
5488 vsi->tc_config.tc_info[i].netdev_tc,
5489 vsi->tc_config.tc_info[i].qcount,
5490 vsi->tc_config.tc_info[i].qoffset);
5493 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5496 /* Assign UP2TC map for the VSI */
5497 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5498 /* Get the actual TC# for the UP */
5499 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5500 /* Get the mapped netdev TC# for the UP */
5501 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5502 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5507 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5508 * @vsi: the VSI being configured
5509 * @ctxt: the ctxt buffer returned from AQ VSI update param command
5511 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5512 struct i40e_vsi_context *ctxt)
5514 /* copy just the sections touched not the entire info
5515 * since not all sections are valid as returned by
5518 vsi->info.mapping_flags = ctxt->info.mapping_flags;
5519 memcpy(&vsi->info.queue_mapping,
5520 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5521 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5522 sizeof(vsi->info.tc_mapping));
5526 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5527 * @vsi: the VSI being reconfigured
5528 * @vsi_offset: offset from main VF VSI
5530 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5532 struct i40e_vsi_context ctxt = {};
5538 return I40E_ERR_PARAM;
5542 ctxt.seid = vsi->seid;
5543 ctxt.pf_num = hw->pf_id;
5544 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5545 ctxt.uplink_seid = vsi->uplink_seid;
5546 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5547 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5548 ctxt.info = vsi->info;
5550 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5552 if (vsi->reconfig_rss) {
5553 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5554 vsi->num_queue_pairs);
5555 ret = i40e_vsi_config_rss(vsi);
5557 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5560 vsi->reconfig_rss = false;
5563 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5565 dev_info(&pf->pdev->dev, "Update vsi config failed, err %s aq_err %s\n",
5566 i40e_stat_str(hw, ret),
5567 i40e_aq_str(hw, hw->aq.asq_last_status));
5570 /* update the local VSI info with updated queue map */
5571 i40e_vsi_update_queue_map(vsi, &ctxt);
5572 vsi->info.valid_sections = 0;
5578 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5579 * @vsi: VSI to be configured
5580 * @enabled_tc: TC bitmap
5582 * This configures a particular VSI for TCs that are mapped to the
5583 * given TC bitmap. It uses default bandwidth share for TCs across
5584 * VSIs to configure TC for a particular VSI.
5587 * It is expected that the VSI queues have been quisced before calling
5590 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5592 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5593 struct i40e_pf *pf = vsi->back;
5594 struct i40e_hw *hw = &pf->hw;
5595 struct i40e_vsi_context ctxt;
5599 /* Check if enabled_tc is same as existing or new TCs */
5600 if (vsi->tc_config.enabled_tc == enabled_tc &&
5601 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5604 /* Enable ETS TCs with equal BW Share for now across all VSIs */
5605 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5606 if (enabled_tc & BIT(i))
5610 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5612 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5614 dev_info(&pf->pdev->dev,
5615 "Failed configuring TC map %d for VSI %d\n",
5616 enabled_tc, vsi->seid);
5617 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5620 dev_info(&pf->pdev->dev,
5621 "Failed querying vsi bw info, err %s aq_err %s\n",
5622 i40e_stat_str(hw, ret),
5623 i40e_aq_str(hw, hw->aq.asq_last_status));
5626 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5627 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5630 valid_tc = bw_config.tc_valid_bits;
5631 /* Always enable TC0, no matter what */
5633 dev_info(&pf->pdev->dev,
5634 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5635 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5636 enabled_tc = valid_tc;
5639 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5641 dev_err(&pf->pdev->dev,
5642 "Unable to configure TC map %d for VSI %d\n",
5643 enabled_tc, vsi->seid);
5648 /* Update Queue Pairs Mapping for currently enabled UPs */
5649 ctxt.seid = vsi->seid;
5650 ctxt.pf_num = vsi->back->hw.pf_id;
5652 ctxt.uplink_seid = vsi->uplink_seid;
5653 ctxt.info = vsi->info;
5654 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5655 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5659 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5662 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5665 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5666 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5667 vsi->num_queue_pairs);
5668 ret = i40e_vsi_config_rss(vsi);
5670 dev_info(&vsi->back->pdev->dev,
5671 "Failed to reconfig rss for num_queues\n");
5674 vsi->reconfig_rss = false;
5676 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5677 ctxt.info.valid_sections |=
5678 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5679 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5682 /* Update the VSI after updating the VSI queue-mapping
5685 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5687 dev_info(&pf->pdev->dev,
5688 "Update vsi tc config failed, err %s aq_err %s\n",
5689 i40e_stat_str(hw, ret),
5690 i40e_aq_str(hw, hw->aq.asq_last_status));
5693 /* update the local VSI info with updated queue map */
5694 i40e_vsi_update_queue_map(vsi, &ctxt);
5695 vsi->info.valid_sections = 0;
5697 /* Update current VSI BW information */
5698 ret = i40e_vsi_get_bw_info(vsi);
5700 dev_info(&pf->pdev->dev,
5701 "Failed updating vsi bw info, err %s aq_err %s\n",
5702 i40e_stat_str(hw, ret),
5703 i40e_aq_str(hw, hw->aq.asq_last_status));
5707 /* Update the netdev TC setup */
5708 i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5714 * i40e_get_link_speed - Returns link speed for the interface
5715 * @vsi: VSI to be configured
5718 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5720 struct i40e_pf *pf = vsi->back;
5722 switch (pf->hw.phy.link_info.link_speed) {
5723 case I40E_LINK_SPEED_40GB:
5725 case I40E_LINK_SPEED_25GB:
5727 case I40E_LINK_SPEED_20GB:
5729 case I40E_LINK_SPEED_10GB:
5731 case I40E_LINK_SPEED_1GB:
5739 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5740 * @vsi: Pointer to vsi structure
5741 * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5743 * Helper function to convert units before send to set BW limit
5745 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5747 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5748 dev_warn(&vsi->back->pdev->dev,
5749 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5750 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5752 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5759 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5760 * @vsi: VSI to be configured
5761 * @seid: seid of the channel/VSI
5762 * @max_tx_rate: max TX rate to be configured as BW limit
5764 * Helper function to set BW limit for a given VSI
5766 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5768 struct i40e_pf *pf = vsi->back;
5773 speed = i40e_get_link_speed(vsi);
5774 if (max_tx_rate > speed) {
5775 dev_err(&pf->pdev->dev,
5776 "Invalid max tx rate %llu specified for VSI seid %d.",
5780 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5781 dev_warn(&pf->pdev->dev,
5782 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5783 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5786 /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5787 credits = max_tx_rate;
5788 do_div(credits, I40E_BW_CREDIT_DIVISOR);
5789 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5790 I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5792 dev_err(&pf->pdev->dev,
5793 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5794 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5795 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5800 * i40e_remove_queue_channels - Remove queue channels for the TCs
5801 * @vsi: VSI to be configured
5803 * Remove queue channels for the TCs
5805 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5807 enum i40e_admin_queue_err last_aq_status;
5808 struct i40e_cloud_filter *cfilter;
5809 struct i40e_channel *ch, *ch_tmp;
5810 struct i40e_pf *pf = vsi->back;
5811 struct hlist_node *node;
5814 /* Reset rss size that was stored when reconfiguring rss for
5815 * channel VSIs with non-power-of-2 queue count.
5817 vsi->current_rss_size = 0;
5819 /* perform cleanup for channels if they exist */
5820 if (list_empty(&vsi->ch_list))
5823 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5824 struct i40e_vsi *p_vsi;
5826 list_del(&ch->list);
5827 p_vsi = ch->parent_vsi;
5828 if (!p_vsi || !ch->initialized) {
5832 /* Reset queue contexts */
5833 for (i = 0; i < ch->num_queue_pairs; i++) {
5834 struct i40e_ring *tx_ring, *rx_ring;
5837 pf_q = ch->base_queue + i;
5838 tx_ring = vsi->tx_rings[pf_q];
5841 rx_ring = vsi->rx_rings[pf_q];
5845 /* Reset BW configured for this VSI via mqprio */
5846 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5848 dev_info(&vsi->back->pdev->dev,
5849 "Failed to reset tx rate for ch->seid %u\n",
5852 /* delete cloud filters associated with this channel */
5853 hlist_for_each_entry_safe(cfilter, node,
5854 &pf->cloud_filter_list, cloud_node) {
5855 if (cfilter->seid != ch->seid)
5858 hash_del(&cfilter->cloud_node);
5859 if (cfilter->dst_port)
5860 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5864 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5866 last_aq_status = pf->hw.aq.asq_last_status;
5868 dev_info(&pf->pdev->dev,
5869 "Failed to delete cloud filter, err %s aq_err %s\n",
5870 i40e_stat_str(&pf->hw, ret),
5871 i40e_aq_str(&pf->hw, last_aq_status));
5875 /* delete VSI from FW */
5876 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5879 dev_err(&vsi->back->pdev->dev,
5880 "unable to remove channel (%d) for parent VSI(%d)\n",
5881 ch->seid, p_vsi->seid);
5884 INIT_LIST_HEAD(&vsi->ch_list);
5888 * i40e_get_max_queues_for_channel
5889 * @vsi: ptr to VSI to which channels are associated with
5891 * Helper function which returns max value among the queue counts set on the
5892 * channels/TCs created.
5894 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5896 struct i40e_channel *ch, *ch_tmp;
5899 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5900 if (!ch->initialized)
5902 if (ch->num_queue_pairs > max)
5903 max = ch->num_queue_pairs;
5910 * i40e_validate_num_queues - validate num_queues w.r.t channel
5911 * @pf: ptr to PF device
5912 * @num_queues: number of queues
5913 * @vsi: the parent VSI
5914 * @reconfig_rss: indicates should the RSS be reconfigured or not
5916 * This function validates number of queues in the context of new channel
5917 * which is being established and determines if RSS should be reconfigured
5918 * or not for parent VSI.
5920 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5921 struct i40e_vsi *vsi, bool *reconfig_rss)
5928 *reconfig_rss = false;
5929 if (vsi->current_rss_size) {
5930 if (num_queues > vsi->current_rss_size) {
5931 dev_dbg(&pf->pdev->dev,
5932 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5933 num_queues, vsi->current_rss_size);
5935 } else if ((num_queues < vsi->current_rss_size) &&
5936 (!is_power_of_2(num_queues))) {
5937 dev_dbg(&pf->pdev->dev,
5938 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5939 num_queues, vsi->current_rss_size);
5944 if (!is_power_of_2(num_queues)) {
5945 /* Find the max num_queues configured for channel if channel
5947 * if channel exist, then enforce 'num_queues' to be more than
5948 * max ever queues configured for channel.
5950 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5951 if (num_queues < max_ch_queues) {
5952 dev_dbg(&pf->pdev->dev,
5953 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5954 num_queues, max_ch_queues);
5957 *reconfig_rss = true;
5964 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5965 * @vsi: the VSI being setup
5966 * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5968 * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5970 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5972 struct i40e_pf *pf = vsi->back;
5973 u8 seed[I40E_HKEY_ARRAY_SIZE];
5974 struct i40e_hw *hw = &pf->hw;
5982 if (rss_size > vsi->rss_size)
5985 local_rss_size = min_t(int, vsi->rss_size, rss_size);
5986 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5990 /* Ignoring user configured lut if there is one */
5991 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5993 /* Use user configured hash key if there is one, otherwise
5996 if (vsi->rss_hkey_user)
5997 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5999 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6001 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6003 dev_info(&pf->pdev->dev,
6004 "Cannot set RSS lut, err %s aq_err %s\n",
6005 i40e_stat_str(hw, ret),
6006 i40e_aq_str(hw, hw->aq.asq_last_status));
6012 /* Do the update w.r.t. storing rss_size */
6013 if (!vsi->orig_rss_size)
6014 vsi->orig_rss_size = vsi->rss_size;
6015 vsi->current_rss_size = local_rss_size;
6021 * i40e_channel_setup_queue_map - Setup a channel queue map
6022 * @pf: ptr to PF device
6023 * @ctxt: VSI context structure
6024 * @ch: ptr to channel structure
6026 * Setup queue map for a specific channel
6028 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6029 struct i40e_vsi_context *ctxt,
6030 struct i40e_channel *ch)
6032 u16 qcount, qmap, sections = 0;
6036 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6037 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6039 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6040 ch->num_queue_pairs = qcount;
6042 /* find the next higher power-of-2 of num queue pairs */
6043 pow = ilog2(qcount);
6044 if (!is_power_of_2(qcount))
6047 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6048 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6050 /* Setup queue TC[0].qmap for given VSI context */
6051 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6053 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6054 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6055 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6056 ctxt->info.valid_sections |= cpu_to_le16(sections);
6060 * i40e_add_channel - add a channel by adding VSI
6061 * @pf: ptr to PF device
6062 * @uplink_seid: underlying HW switching element (VEB) ID
6063 * @ch: ptr to channel structure
6065 * Add a channel (VSI) using add_vsi and queue_map
6067 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6068 struct i40e_channel *ch)
6070 struct i40e_hw *hw = &pf->hw;
6071 struct i40e_vsi_context ctxt;
6072 u8 enabled_tc = 0x1; /* TC0 enabled */
6075 if (ch->type != I40E_VSI_VMDQ2) {
6076 dev_info(&pf->pdev->dev,
6077 "add new vsi failed, ch->type %d\n", ch->type);
6081 memset(&ctxt, 0, sizeof(ctxt));
6082 ctxt.pf_num = hw->pf_id;
6084 ctxt.uplink_seid = uplink_seid;
6085 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6086 if (ch->type == I40E_VSI_VMDQ2)
6087 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6089 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
6090 ctxt.info.valid_sections |=
6091 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6092 ctxt.info.switch_id =
6093 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6096 /* Set queue map for a given VSI context */
6097 i40e_channel_setup_queue_map(pf, &ctxt, ch);
6099 /* Now time to create VSI */
6100 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6102 dev_info(&pf->pdev->dev,
6103 "add new vsi failed, err %s aq_err %s\n",
6104 i40e_stat_str(&pf->hw, ret),
6105 i40e_aq_str(&pf->hw,
6106 pf->hw.aq.asq_last_status));
6110 /* Success, update channel, set enabled_tc only if the channel
6113 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6114 ch->seid = ctxt.seid;
6115 ch->vsi_number = ctxt.vsi_number;
6116 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6118 /* copy just the sections touched not the entire info
6119 * since not all sections are valid as returned by
6122 ch->info.mapping_flags = ctxt.info.mapping_flags;
6123 memcpy(&ch->info.queue_mapping,
6124 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6125 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6126 sizeof(ctxt.info.tc_mapping));
6131 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6134 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6138 bw_data.tc_valid_bits = ch->enabled_tc;
6139 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6140 bw_data.tc_bw_credits[i] = bw_share[i];
6142 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6145 dev_info(&vsi->back->pdev->dev,
6146 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6147 vsi->back->hw.aq.asq_last_status, ch->seid);
6151 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6152 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6158 * i40e_channel_config_tx_ring - config TX ring associated with new channel
6159 * @pf: ptr to PF device
6160 * @vsi: the VSI being setup
6161 * @ch: ptr to channel structure
6163 * Configure TX rings associated with channel (VSI) since queues are being
6166 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6167 struct i40e_vsi *vsi,
6168 struct i40e_channel *ch)
6172 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6174 /* Enable ETS TCs with equal BW Share for now across all VSIs */
6175 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6176 if (ch->enabled_tc & BIT(i))
6180 /* configure BW for new VSI */
6181 ret = i40e_channel_config_bw(vsi, ch, bw_share);
6183 dev_info(&vsi->back->pdev->dev,
6184 "Failed configuring TC map %d for channel (seid %u)\n",
6185 ch->enabled_tc, ch->seid);
6189 for (i = 0; i < ch->num_queue_pairs; i++) {
6190 struct i40e_ring *tx_ring, *rx_ring;
6193 pf_q = ch->base_queue + i;
6195 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6198 tx_ring = vsi->tx_rings[pf_q];
6201 /* Get the RX ring ptr */
6202 rx_ring = vsi->rx_rings[pf_q];
6210 * i40e_setup_hw_channel - setup new channel
6211 * @pf: ptr to PF device
6212 * @vsi: the VSI being setup
6213 * @ch: ptr to channel structure
6214 * @uplink_seid: underlying HW switching element (VEB) ID
6215 * @type: type of channel to be created (VMDq2/VF)
6217 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6218 * and configures TX rings accordingly
6220 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6221 struct i40e_vsi *vsi,
6222 struct i40e_channel *ch,
6223 u16 uplink_seid, u8 type)
6227 ch->initialized = false;
6228 ch->base_queue = vsi->next_base_queue;
6231 /* Proceed with creation of channel (VMDq2) VSI */
6232 ret = i40e_add_channel(pf, uplink_seid, ch);
6234 dev_info(&pf->pdev->dev,
6235 "failed to add_channel using uplink_seid %u\n",
6240 /* Mark the successful creation of channel */
6241 ch->initialized = true;
6243 /* Reconfigure TX queues using QTX_CTL register */
6244 ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6246 dev_info(&pf->pdev->dev,
6247 "failed to configure TX rings for channel %u\n",
6252 /* update 'next_base_queue' */
6253 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6254 dev_dbg(&pf->pdev->dev,
6255 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6256 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6257 ch->num_queue_pairs,
6258 vsi->next_base_queue);
6263 * i40e_setup_channel - setup new channel using uplink element
6264 * @pf: ptr to PF device
6265 * @vsi: pointer to the VSI to set up the channel within
6266 * @ch: ptr to channel structure
6268 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6269 * and uplink switching element (uplink_seid)
6271 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6272 struct i40e_channel *ch)
6278 if (vsi->type == I40E_VSI_MAIN) {
6279 vsi_type = I40E_VSI_VMDQ2;
6281 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6286 /* underlying switching element */
6287 seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6289 /* create channel (VSI), configure TX rings */
6290 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6292 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6296 return ch->initialized ? true : false;
6300 * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6301 * @vsi: ptr to VSI which has PF backing
6303 * Sets up switch mode correctly if it needs to be changed and perform
6304 * what are allowed modes.
6306 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6309 struct i40e_pf *pf = vsi->back;
6310 struct i40e_hw *hw = &pf->hw;
6313 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6317 if (hw->dev_caps.switch_mode) {
6318 /* if switch mode is set, support mode2 (non-tunneled for
6319 * cloud filter) for now
6321 u32 switch_mode = hw->dev_caps.switch_mode &
6322 I40E_SWITCH_MODE_MASK;
6323 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6324 if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6326 dev_err(&pf->pdev->dev,
6327 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6328 hw->dev_caps.switch_mode);
6333 /* Set Bit 7 to be valid */
6334 mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6336 /* Set L4type for TCP support */
6337 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6339 /* Set cloud filter mode */
6340 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6342 /* Prep mode field for set_switch_config */
6343 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6344 pf->last_sw_conf_valid_flags,
6346 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6347 dev_err(&pf->pdev->dev,
6348 "couldn't set switch config bits, err %s aq_err %s\n",
6349 i40e_stat_str(hw, ret),
6351 hw->aq.asq_last_status));
6357 * i40e_create_queue_channel - function to create channel
6358 * @vsi: VSI to be configured
6359 * @ch: ptr to channel (it contains channel specific params)
6361 * This function creates channel (VSI) using num_queues specified by user,
6362 * reconfigs RSS if needed.
6364 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6365 struct i40e_channel *ch)
6367 struct i40e_pf *pf = vsi->back;
6374 if (!ch->num_queue_pairs) {
6375 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6376 ch->num_queue_pairs);
6380 /* validate user requested num_queues for channel */
6381 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6384 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6385 ch->num_queue_pairs);
6389 /* By default we are in VEPA mode, if this is the first VF/VMDq
6390 * VSI to be added switch to VEB mode.
6393 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6394 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6396 if (vsi->type == I40E_VSI_MAIN) {
6397 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6398 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6400 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6402 /* now onwards for main VSI, number of queues will be value
6403 * of TC0's queue count
6407 /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6408 * it should be more than num_queues
6410 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6411 dev_dbg(&pf->pdev->dev,
6412 "Error: cnt_q_avail (%u) less than num_queues %d\n",
6413 vsi->cnt_q_avail, ch->num_queue_pairs);
6417 /* reconfig_rss only if vsi type is MAIN_VSI */
6418 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6419 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6421 dev_info(&pf->pdev->dev,
6422 "Error: unable to reconfig rss for num_queues (%u)\n",
6423 ch->num_queue_pairs);
6428 if (!i40e_setup_channel(pf, vsi, ch)) {
6429 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6433 dev_info(&pf->pdev->dev,
6434 "Setup channel (id:%u) utilizing num_queues %d\n",
6435 ch->seid, ch->num_queue_pairs);
6437 /* configure VSI for BW limit */
6438 if (ch->max_tx_rate) {
6439 u64 credits = ch->max_tx_rate;
6441 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6444 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6445 dev_dbg(&pf->pdev->dev,
6446 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6452 /* in case of VF, this will be main SRIOV VSI */
6453 ch->parent_vsi = vsi;
6455 /* and update main_vsi's count for queue_available to use */
6456 vsi->cnt_q_avail -= ch->num_queue_pairs;
6462 * i40e_configure_queue_channels - Add queue channel for the given TCs
6463 * @vsi: VSI to be configured
6465 * Configures queue channel mapping to the given TCs
6467 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6469 struct i40e_channel *ch;
6473 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6474 vsi->tc_seid_map[0] = vsi->seid;
6475 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6476 if (vsi->tc_config.enabled_tc & BIT(i)) {
6477 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6483 INIT_LIST_HEAD(&ch->list);
6484 ch->num_queue_pairs =
6485 vsi->tc_config.tc_info[i].qcount;
6487 vsi->tc_config.tc_info[i].qoffset;
6489 /* Bandwidth limit through tc interface is in bytes/s,
6492 max_rate = vsi->mqprio_qopt.max_rate[i];
6493 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6494 ch->max_tx_rate = max_rate;
6496 list_add_tail(&ch->list, &vsi->ch_list);
6498 ret = i40e_create_queue_channel(vsi, ch);
6500 dev_err(&vsi->back->pdev->dev,
6501 "Failed creating queue channel with TC%d: queues %d\n",
6502 i, ch->num_queue_pairs);
6505 vsi->tc_seid_map[i] = ch->seid;
6511 i40e_remove_queue_channels(vsi);
6516 * i40e_veb_config_tc - Configure TCs for given VEB
6518 * @enabled_tc: TC bitmap
6520 * Configures given TC bitmap for VEB (switching) element
6522 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6524 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6525 struct i40e_pf *pf = veb->pf;
6529 /* No TCs or already enabled TCs just return */
6530 if (!enabled_tc || veb->enabled_tc == enabled_tc)
6533 bw_data.tc_valid_bits = enabled_tc;
6534 /* bw_data.absolute_credits is not set (relative) */
6536 /* Enable ETS TCs with equal BW Share for now */
6537 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6538 if (enabled_tc & BIT(i))
6539 bw_data.tc_bw_share_credits[i] = 1;
6542 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6545 dev_info(&pf->pdev->dev,
6546 "VEB bw config failed, err %s aq_err %s\n",
6547 i40e_stat_str(&pf->hw, ret),
6548 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6552 /* Update the BW information */
6553 ret = i40e_veb_get_bw_info(veb);
6555 dev_info(&pf->pdev->dev,
6556 "Failed getting veb bw config, err %s aq_err %s\n",
6557 i40e_stat_str(&pf->hw, ret),
6558 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6565 #ifdef CONFIG_I40E_DCB
6567 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6570 * Reconfigure VEB/VSIs on a given PF; it is assumed that
6571 * the caller would've quiesce all the VSIs before calling
6574 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6580 /* Enable the TCs available on PF to all VEBs */
6581 tc_map = i40e_pf_get_tc_map(pf);
6582 for (v = 0; v < I40E_MAX_VEB; v++) {
6585 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6587 dev_info(&pf->pdev->dev,
6588 "Failed configuring TC for VEB seid=%d\n",
6590 /* Will try to configure as many components */
6594 /* Update each VSI */
6595 for (v = 0; v < pf->num_alloc_vsi; v++) {
6599 /* - Enable all TCs for the LAN VSI
6600 * - For all others keep them at TC0 for now
6602 if (v == pf->lan_vsi)
6603 tc_map = i40e_pf_get_tc_map(pf);
6605 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6607 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6609 dev_info(&pf->pdev->dev,
6610 "Failed configuring TC for VSI seid=%d\n",
6612 /* Will try to configure as many components */
6614 /* Re-configure VSI vectors based on updated TC map */
6615 i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6616 if (pf->vsi[v]->netdev)
6617 i40e_dcbnl_set_all(pf->vsi[v]);
6623 * i40e_resume_port_tx - Resume port Tx
6626 * Resume a port's Tx and issue a PF reset in case of failure to
6629 static int i40e_resume_port_tx(struct i40e_pf *pf)
6631 struct i40e_hw *hw = &pf->hw;
6634 ret = i40e_aq_resume_port_tx(hw, NULL);
6636 dev_info(&pf->pdev->dev,
6637 "Resume Port Tx failed, err %s aq_err %s\n",
6638 i40e_stat_str(&pf->hw, ret),
6639 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6640 /* Schedule PF reset to recover */
6641 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6642 i40e_service_event_schedule(pf);
6649 * i40e_init_pf_dcb - Initialize DCB configuration
6650 * @pf: PF being configured
6652 * Query the current DCB configuration and cache it
6653 * in the hardware structure
6655 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6657 struct i40e_hw *hw = &pf->hw;
6660 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6661 * Also do not enable DCBx if FW LLDP agent is disabled
6663 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
6664 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) {
6665 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n");
6666 err = I40E_NOT_SUPPORTED;
6670 err = i40e_init_dcb(hw, true);
6672 /* Device/Function is not DCBX capable */
6673 if ((!hw->func_caps.dcb) ||
6674 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6675 dev_info(&pf->pdev->dev,
6676 "DCBX offload is not supported or is disabled for this PF.\n");
6678 /* When status is not DISABLED then DCBX in FW */
6679 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6680 DCB_CAP_DCBX_VER_IEEE;
6682 pf->flags |= I40E_FLAG_DCB_CAPABLE;
6683 /* Enable DCB tagging only when more than one TC
6684 * or explicitly disable if only one TC
6686 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6687 pf->flags |= I40E_FLAG_DCB_ENABLED;
6689 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6690 dev_dbg(&pf->pdev->dev,
6691 "DCBX offload is supported for this PF.\n");
6693 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6694 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6695 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6697 dev_info(&pf->pdev->dev,
6698 "Query for DCB configuration failed, err %s aq_err %s\n",
6699 i40e_stat_str(&pf->hw, err),
6700 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6706 #endif /* CONFIG_I40E_DCB */
6709 * i40e_print_link_message - print link up or down
6710 * @vsi: the VSI for which link needs a message
6711 * @isup: true of link is up, false otherwise
6713 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6715 enum i40e_aq_link_speed new_speed;
6716 struct i40e_pf *pf = vsi->back;
6717 char *speed = "Unknown";
6718 char *fc = "Unknown";
6724 new_speed = pf->hw.phy.link_info.link_speed;
6726 new_speed = I40E_LINK_SPEED_UNKNOWN;
6728 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6730 vsi->current_isup = isup;
6731 vsi->current_speed = new_speed;
6733 netdev_info(vsi->netdev, "NIC Link is Down\n");
6737 /* Warn user if link speed on NPAR enabled partition is not at
6740 if (pf->hw.func_caps.npar_enable &&
6741 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6742 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6743 netdev_warn(vsi->netdev,
6744 "The partition detected link speed that is less than 10Gbps\n");
6746 switch (pf->hw.phy.link_info.link_speed) {
6747 case I40E_LINK_SPEED_40GB:
6750 case I40E_LINK_SPEED_20GB:
6753 case I40E_LINK_SPEED_25GB:
6756 case I40E_LINK_SPEED_10GB:
6759 case I40E_LINK_SPEED_5GB:
6762 case I40E_LINK_SPEED_2_5GB:
6765 case I40E_LINK_SPEED_1GB:
6768 case I40E_LINK_SPEED_100MB:
6775 switch (pf->hw.fc.current_mode) {
6779 case I40E_FC_TX_PAUSE:
6782 case I40E_FC_RX_PAUSE:
6790 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6795 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6798 if (pf->hw.phy.link_info.fec_info &
6799 I40E_AQ_CONFIG_FEC_KR_ENA)
6800 fec = "CL74 FC-FEC/BASE-R";
6801 else if (pf->hw.phy.link_info.fec_info &
6802 I40E_AQ_CONFIG_FEC_RS_ENA)
6803 fec = "CL108 RS-FEC";
6805 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
6806 * both RS and FC are requested
6808 if (vsi->back->hw.phy.link_info.req_fec_info &
6809 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
6810 if (vsi->back->hw.phy.link_info.req_fec_info &
6811 I40E_AQ_REQUEST_FEC_RS)
6812 req_fec = "CL108 RS-FEC";
6814 req_fec = "CL74 FC-FEC/BASE-R";
6816 netdev_info(vsi->netdev,
6817 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
6818 speed, req_fec, fec, an, fc);
6819 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
6824 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6827 if (pf->hw.phy.link_info.fec_info &
6828 I40E_AQ_CONFIG_FEC_KR_ENA)
6829 fec = "CL74 FC-FEC/BASE-R";
6831 if (pf->hw.phy.link_info.req_fec_info &
6832 I40E_AQ_REQUEST_FEC_KR)
6833 req_fec = "CL74 FC-FEC/BASE-R";
6835 netdev_info(vsi->netdev,
6836 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
6837 speed, req_fec, fec, an, fc);
6839 netdev_info(vsi->netdev,
6840 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
6847 * i40e_up_complete - Finish the last steps of bringing up a connection
6848 * @vsi: the VSI being configured
6850 static int i40e_up_complete(struct i40e_vsi *vsi)
6852 struct i40e_pf *pf = vsi->back;
6855 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6856 i40e_vsi_configure_msix(vsi);
6858 i40e_configure_msi_and_legacy(vsi);
6861 err = i40e_vsi_start_rings(vsi);
6865 clear_bit(__I40E_VSI_DOWN, vsi->state);
6866 i40e_napi_enable_all(vsi);
6867 i40e_vsi_enable_irq(vsi);
6869 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
6871 i40e_print_link_message(vsi, true);
6872 netif_tx_start_all_queues(vsi->netdev);
6873 netif_carrier_on(vsi->netdev);
6876 /* replay FDIR SB filters */
6877 if (vsi->type == I40E_VSI_FDIR) {
6878 /* reset fd counters */
6881 i40e_fdir_filter_restore(vsi);
6884 /* On the next run of the service_task, notify any clients of the new
6887 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6888 i40e_service_event_schedule(pf);
6894 * i40e_vsi_reinit_locked - Reset the VSI
6895 * @vsi: the VSI being configured
6897 * Rebuild the ring structs after some configuration
6898 * has changed, e.g. MTU size.
6900 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
6902 struct i40e_pf *pf = vsi->back;
6904 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
6905 usleep_range(1000, 2000);
6909 clear_bit(__I40E_CONFIG_BUSY, pf->state);
6913 * i40e_force_link_state - Force the link status
6914 * @pf: board private structure
6915 * @is_up: whether the link state should be forced up or down
6917 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
6919 struct i40e_aq_get_phy_abilities_resp abilities;
6920 struct i40e_aq_set_phy_config config = {0};
6921 bool non_zero_phy_type = is_up;
6922 struct i40e_hw *hw = &pf->hw;
6927 /* Card might've been put in an unstable state by other drivers
6928 * and applications, which causes incorrect speed values being
6929 * set on startup. In order to clear speed registers, we call
6930 * get_phy_capabilities twice, once to get initial state of
6931 * available speeds, and once to get current PHY config.
6933 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
6936 dev_err(&pf->pdev->dev,
6937 "failed to get phy cap., ret = %s last_status = %s\n",
6938 i40e_stat_str(hw, err),
6939 i40e_aq_str(hw, hw->aq.asq_last_status));
6942 speed = abilities.link_speed;
6944 /* Get the current phy config */
6945 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
6948 dev_err(&pf->pdev->dev,
6949 "failed to get phy cap., ret = %s last_status = %s\n",
6950 i40e_stat_str(hw, err),
6951 i40e_aq_str(hw, hw->aq.asq_last_status));
6955 /* If link needs to go up, but was not forced to go down,
6956 * and its speed values are OK, no need for a flap
6957 * if non_zero_phy_type was set, still need to force up
6959 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
6960 non_zero_phy_type = true;
6961 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
6962 return I40E_SUCCESS;
6964 /* To force link we need to set bits for all supported PHY types,
6965 * but there are now more than 32, so we need to split the bitmap
6966 * across two fields.
6968 mask = I40E_PHY_TYPES_BITMASK;
6970 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
6971 config.phy_type_ext =
6972 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
6973 /* Copy the old settings, except of phy_type */
6974 config.abilities = abilities.abilities;
6975 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
6977 config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
6979 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
6981 if (abilities.link_speed != 0)
6982 config.link_speed = abilities.link_speed;
6984 config.link_speed = speed;
6985 config.eee_capability = abilities.eee_capability;
6986 config.eeer = abilities.eeer_val;
6987 config.low_power_ctrl = abilities.d3_lpan;
6988 config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
6989 I40E_AQ_PHY_FEC_CONFIG_MASK;
6990 err = i40e_aq_set_phy_config(hw, &config, NULL);
6993 dev_err(&pf->pdev->dev,
6994 "set phy config ret = %s last_status = %s\n",
6995 i40e_stat_str(&pf->hw, err),
6996 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7000 /* Update the link info */
7001 err = i40e_update_link_info(hw);
7003 /* Wait a little bit (on 40G cards it sometimes takes a really
7004 * long time for link to come back from the atomic reset)
7008 i40e_update_link_info(hw);
7011 i40e_aq_set_link_restart_an(hw, is_up, NULL);
7013 return I40E_SUCCESS;
7017 * i40e_up - Bring the connection back up after being down
7018 * @vsi: the VSI being configured
7020 int i40e_up(struct i40e_vsi *vsi)
7024 if (vsi->type == I40E_VSI_MAIN &&
7025 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7026 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7027 i40e_force_link_state(vsi->back, true);
7029 err = i40e_vsi_configure(vsi);
7031 err = i40e_up_complete(vsi);
7037 * i40e_down - Shutdown the connection processing
7038 * @vsi: the VSI being stopped
7040 void i40e_down(struct i40e_vsi *vsi)
7044 /* It is assumed that the caller of this function
7045 * sets the vsi->state __I40E_VSI_DOWN bit.
7048 netif_carrier_off(vsi->netdev);
7049 netif_tx_disable(vsi->netdev);
7051 i40e_vsi_disable_irq(vsi);
7052 i40e_vsi_stop_rings(vsi);
7053 if (vsi->type == I40E_VSI_MAIN &&
7054 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7055 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7056 i40e_force_link_state(vsi->back, false);
7057 i40e_napi_disable_all(vsi);
7059 for (i = 0; i < vsi->num_queue_pairs; i++) {
7060 i40e_clean_tx_ring(vsi->tx_rings[i]);
7061 if (i40e_enabled_xdp_vsi(vsi)) {
7062 /* Make sure that in-progress ndo_xdp_xmit and
7063 * ndo_xsk_wakeup calls are completed.
7066 i40e_clean_tx_ring(vsi->xdp_rings[i]);
7068 i40e_clean_rx_ring(vsi->rx_rings[i]);
7074 * i40e_validate_mqprio_qopt- validate queue mapping info
7075 * @vsi: the VSI being configured
7076 * @mqprio_qopt: queue parametrs
7078 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7079 struct tc_mqprio_qopt_offload *mqprio_qopt)
7081 u64 sum_max_rate = 0;
7085 if (mqprio_qopt->qopt.offset[0] != 0 ||
7086 mqprio_qopt->qopt.num_tc < 1 ||
7087 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7089 for (i = 0; ; i++) {
7090 if (!mqprio_qopt->qopt.count[i])
7092 if (mqprio_qopt->min_rate[i]) {
7093 dev_err(&vsi->back->pdev->dev,
7094 "Invalid min tx rate (greater than 0) specified\n");
7097 max_rate = mqprio_qopt->max_rate[i];
7098 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7099 sum_max_rate += max_rate;
7101 if (i >= mqprio_qopt->qopt.num_tc - 1)
7103 if (mqprio_qopt->qopt.offset[i + 1] !=
7104 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7107 if (vsi->num_queue_pairs <
7108 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7109 dev_err(&vsi->back->pdev->dev,
7110 "Failed to create traffic channel, insufficient number of queues.\n");
7113 if (sum_max_rate > i40e_get_link_speed(vsi)) {
7114 dev_err(&vsi->back->pdev->dev,
7115 "Invalid max tx rate specified\n");
7122 * i40e_vsi_set_default_tc_config - set default values for tc configuration
7123 * @vsi: the VSI being configured
7125 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7130 /* Only TC0 is enabled */
7131 vsi->tc_config.numtc = 1;
7132 vsi->tc_config.enabled_tc = 1;
7133 qcount = min_t(int, vsi->alloc_queue_pairs,
7134 i40e_pf_get_max_q_per_tc(vsi->back));
7135 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7136 /* For the TC that is not enabled set the offset to to default
7137 * queue and allocate one queue for the given TC.
7139 vsi->tc_config.tc_info[i].qoffset = 0;
7141 vsi->tc_config.tc_info[i].qcount = qcount;
7143 vsi->tc_config.tc_info[i].qcount = 1;
7144 vsi->tc_config.tc_info[i].netdev_tc = 0;
7149 * i40e_del_macvlan_filter
7150 * @hw: pointer to the HW structure
7151 * @seid: seid of the channel VSI
7152 * @macaddr: the mac address to apply as a filter
7153 * @aq_err: store the admin Q error
7155 * This function deletes a mac filter on the channel VSI which serves as the
7156 * macvlan. Returns 0 on success.
7158 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7159 const u8 *macaddr, int *aq_err)
7161 struct i40e_aqc_remove_macvlan_element_data element;
7164 memset(&element, 0, sizeof(element));
7165 ether_addr_copy(element.mac_addr, macaddr);
7166 element.vlan_tag = 0;
7167 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7168 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7169 *aq_err = hw->aq.asq_last_status;
7175 * i40e_add_macvlan_filter
7176 * @hw: pointer to the HW structure
7177 * @seid: seid of the channel VSI
7178 * @macaddr: the mac address to apply as a filter
7179 * @aq_err: store the admin Q error
7181 * This function adds a mac filter on the channel VSI which serves as the
7182 * macvlan. Returns 0 on success.
7184 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7185 const u8 *macaddr, int *aq_err)
7187 struct i40e_aqc_add_macvlan_element_data element;
7191 ether_addr_copy(element.mac_addr, macaddr);
7192 element.vlan_tag = 0;
7193 element.queue_number = 0;
7194 element.match_method = I40E_AQC_MM_ERR_NO_RES;
7195 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7196 element.flags = cpu_to_le16(cmd_flags);
7197 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7198 *aq_err = hw->aq.asq_last_status;
7204 * i40e_reset_ch_rings - Reset the queue contexts in a channel
7205 * @vsi: the VSI we want to access
7206 * @ch: the channel we want to access
7208 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7210 struct i40e_ring *tx_ring, *rx_ring;
7214 for (i = 0; i < ch->num_queue_pairs; i++) {
7215 pf_q = ch->base_queue + i;
7216 tx_ring = vsi->tx_rings[pf_q];
7218 rx_ring = vsi->rx_rings[pf_q];
7224 * i40e_free_macvlan_channels
7225 * @vsi: the VSI we want to access
7227 * This function frees the Qs of the channel VSI from
7228 * the stack and also deletes the channel VSIs which
7229 * serve as macvlans.
7231 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7233 struct i40e_channel *ch, *ch_tmp;
7236 if (list_empty(&vsi->macvlan_list))
7239 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7240 struct i40e_vsi *parent_vsi;
7242 if (i40e_is_channel_macvlan(ch)) {
7243 i40e_reset_ch_rings(vsi, ch);
7244 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7245 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7246 netdev_set_sb_channel(ch->fwd->netdev, 0);
7251 list_del(&ch->list);
7252 parent_vsi = ch->parent_vsi;
7253 if (!parent_vsi || !ch->initialized) {
7258 /* remove the VSI */
7259 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7262 dev_err(&vsi->back->pdev->dev,
7263 "unable to remove channel (%d) for parent VSI(%d)\n",
7264 ch->seid, parent_vsi->seid);
7267 vsi->macvlan_cnt = 0;
7271 * i40e_fwd_ring_up - bring the macvlan device up
7272 * @vsi: the VSI we want to access
7273 * @vdev: macvlan netdevice
7274 * @fwd: the private fwd structure
7276 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7277 struct i40e_fwd_adapter *fwd)
7279 struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7280 int ret = 0, num_tc = 1, i, aq_err;
7281 struct i40e_pf *pf = vsi->back;
7282 struct i40e_hw *hw = &pf->hw;
7284 /* Go through the list and find an available channel */
7285 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7286 if (!i40e_is_channel_macvlan(iter)) {
7288 /* record configuration for macvlan interface in vdev */
7289 for (i = 0; i < num_tc; i++)
7290 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7292 iter->num_queue_pairs,
7294 for (i = 0; i < iter->num_queue_pairs; i++) {
7295 struct i40e_ring *tx_ring, *rx_ring;
7298 pf_q = iter->base_queue + i;
7300 /* Get to TX ring ptr */
7301 tx_ring = vsi->tx_rings[pf_q];
7304 /* Get the RX ring ptr */
7305 rx_ring = vsi->rx_rings[pf_q];
7316 /* Guarantee all rings are updated before we update the
7317 * MAC address filter.
7321 /* Add a mac filter */
7322 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7324 /* if we cannot add the MAC rule then disable the offload */
7325 macvlan_release_l2fw_offload(vdev);
7326 for (i = 0; i < ch->num_queue_pairs; i++) {
7327 struct i40e_ring *rx_ring;
7330 pf_q = ch->base_queue + i;
7331 rx_ring = vsi->rx_rings[pf_q];
7332 rx_ring->netdev = NULL;
7334 dev_info(&pf->pdev->dev,
7335 "Error adding mac filter on macvlan err %s, aq_err %s\n",
7336 i40e_stat_str(hw, ret),
7337 i40e_aq_str(hw, aq_err));
7338 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7345 * i40e_setup_macvlans - create the channels which will be macvlans
7346 * @vsi: the VSI we want to access
7347 * @macvlan_cnt: no. of macvlans to be setup
7348 * @qcnt: no. of Qs per macvlan
7349 * @vdev: macvlan netdevice
7351 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7352 struct net_device *vdev)
7354 struct i40e_pf *pf = vsi->back;
7355 struct i40e_hw *hw = &pf->hw;
7356 struct i40e_vsi_context ctxt;
7357 u16 sections, qmap, num_qps;
7358 struct i40e_channel *ch;
7359 int i, pow, ret = 0;
7362 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7365 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7367 /* find the next higher power-of-2 of num queue pairs */
7368 pow = fls(roundup_pow_of_two(num_qps) - 1);
7370 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7371 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7373 /* Setup context bits for the main VSI */
7374 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7375 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7376 memset(&ctxt, 0, sizeof(ctxt));
7377 ctxt.seid = vsi->seid;
7378 ctxt.pf_num = vsi->back->hw.pf_id;
7380 ctxt.uplink_seid = vsi->uplink_seid;
7381 ctxt.info = vsi->info;
7382 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7383 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7384 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7385 ctxt.info.valid_sections |= cpu_to_le16(sections);
7387 /* Reconfigure RSS for main VSI with new max queue count */
7388 vsi->rss_size = max_t(u16, num_qps, qcnt);
7389 ret = i40e_vsi_config_rss(vsi);
7391 dev_info(&pf->pdev->dev,
7392 "Failed to reconfig RSS for num_queues (%u)\n",
7396 vsi->reconfig_rss = true;
7397 dev_dbg(&vsi->back->pdev->dev,
7398 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7399 vsi->next_base_queue = num_qps;
7400 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7402 /* Update the VSI after updating the VSI queue-mapping
7405 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7407 dev_info(&pf->pdev->dev,
7408 "Update vsi tc config failed, err %s aq_err %s\n",
7409 i40e_stat_str(hw, ret),
7410 i40e_aq_str(hw, hw->aq.asq_last_status));
7413 /* update the local VSI info with updated queue map */
7414 i40e_vsi_update_queue_map(vsi, &ctxt);
7415 vsi->info.valid_sections = 0;
7417 /* Create channels for macvlans */
7418 INIT_LIST_HEAD(&vsi->macvlan_list);
7419 for (i = 0; i < macvlan_cnt; i++) {
7420 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7425 INIT_LIST_HEAD(&ch->list);
7426 ch->num_queue_pairs = qcnt;
7427 if (!i40e_setup_channel(pf, vsi, ch)) {
7432 ch->parent_vsi = vsi;
7433 vsi->cnt_q_avail -= ch->num_queue_pairs;
7435 list_add_tail(&ch->list, &vsi->macvlan_list);
7441 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7442 i40e_free_macvlan_channels(vsi);
7448 * i40e_fwd_add - configure macvlans
7449 * @netdev: net device to configure
7450 * @vdev: macvlan netdevice
7452 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7454 struct i40e_netdev_priv *np = netdev_priv(netdev);
7455 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7456 struct i40e_vsi *vsi = np->vsi;
7457 struct i40e_pf *pf = vsi->back;
7458 struct i40e_fwd_adapter *fwd;
7459 int avail_macvlan, ret;
7461 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7462 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7463 return ERR_PTR(-EINVAL);
7465 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7466 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7467 return ERR_PTR(-EINVAL);
7469 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7470 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7471 return ERR_PTR(-EINVAL);
7474 /* The macvlan device has to be a single Q device so that the
7475 * tc_to_txq field can be reused to pick the tx queue.
7477 if (netif_is_multiqueue(vdev))
7478 return ERR_PTR(-ERANGE);
7480 if (!vsi->macvlan_cnt) {
7481 /* reserve bit 0 for the pf device */
7482 set_bit(0, vsi->fwd_bitmask);
7484 /* Try to reserve as many queues as possible for macvlans. First
7485 * reserve 3/4th of max vectors, then half, then quarter and
7486 * calculate Qs per macvlan as you go
7488 vectors = pf->num_lan_msix;
7489 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7490 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7492 macvlan_cnt = (vectors - 32) / 4;
7493 } else if (vectors <= 64 && vectors > 32) {
7494 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7496 macvlan_cnt = (vectors - 16) / 2;
7497 } else if (vectors <= 32 && vectors > 16) {
7498 /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7500 macvlan_cnt = vectors - 16;
7501 } else if (vectors <= 16 && vectors > 8) {
7502 /* allocate 1 Q per macvlan and 8 Qs to the PF */
7504 macvlan_cnt = vectors - 8;
7506 /* allocate 1 Q per macvlan and 1 Q to the PF */
7508 macvlan_cnt = vectors - 1;
7511 if (macvlan_cnt == 0)
7512 return ERR_PTR(-EBUSY);
7514 /* Quiesce VSI queues */
7515 i40e_quiesce_vsi(vsi);
7517 /* sets up the macvlans but does not "enable" them */
7518 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7521 return ERR_PTR(ret);
7524 i40e_unquiesce_vsi(vsi);
7526 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7528 if (avail_macvlan >= I40E_MAX_MACVLANS)
7529 return ERR_PTR(-EBUSY);
7531 /* create the fwd struct */
7532 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7534 return ERR_PTR(-ENOMEM);
7536 set_bit(avail_macvlan, vsi->fwd_bitmask);
7537 fwd->bit_no = avail_macvlan;
7538 netdev_set_sb_channel(vdev, avail_macvlan);
7541 if (!netif_running(netdev))
7544 /* Set fwd ring up */
7545 ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7547 /* unbind the queues and drop the subordinate channel config */
7548 netdev_unbind_sb_channel(netdev, vdev);
7549 netdev_set_sb_channel(vdev, 0);
7552 return ERR_PTR(-EINVAL);
7559 * i40e_del_all_macvlans - Delete all the mac filters on the channels
7560 * @vsi: the VSI we want to access
7562 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7564 struct i40e_channel *ch, *ch_tmp;
7565 struct i40e_pf *pf = vsi->back;
7566 struct i40e_hw *hw = &pf->hw;
7567 int aq_err, ret = 0;
7569 if (list_empty(&vsi->macvlan_list))
7572 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7573 if (i40e_is_channel_macvlan(ch)) {
7574 ret = i40e_del_macvlan_filter(hw, ch->seid,
7575 i40e_channel_mac(ch),
7578 /* Reset queue contexts */
7579 i40e_reset_ch_rings(vsi, ch);
7580 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7581 netdev_unbind_sb_channel(vsi->netdev,
7583 netdev_set_sb_channel(ch->fwd->netdev, 0);
7592 * i40e_fwd_del - delete macvlan interfaces
7593 * @netdev: net device to configure
7594 * @vdev: macvlan netdevice
7596 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7598 struct i40e_netdev_priv *np = netdev_priv(netdev);
7599 struct i40e_fwd_adapter *fwd = vdev;
7600 struct i40e_channel *ch, *ch_tmp;
7601 struct i40e_vsi *vsi = np->vsi;
7602 struct i40e_pf *pf = vsi->back;
7603 struct i40e_hw *hw = &pf->hw;
7604 int aq_err, ret = 0;
7606 /* Find the channel associated with the macvlan and del mac filter */
7607 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7608 if (i40e_is_channel_macvlan(ch) &&
7609 ether_addr_equal(i40e_channel_mac(ch),
7610 fwd->netdev->dev_addr)) {
7611 ret = i40e_del_macvlan_filter(hw, ch->seid,
7612 i40e_channel_mac(ch),
7615 /* Reset queue contexts */
7616 i40e_reset_ch_rings(vsi, ch);
7617 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7618 netdev_unbind_sb_channel(netdev, fwd->netdev);
7619 netdev_set_sb_channel(fwd->netdev, 0);
7623 dev_info(&pf->pdev->dev,
7624 "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7625 i40e_stat_str(hw, ret),
7626 i40e_aq_str(hw, aq_err));
7634 * i40e_setup_tc - configure multiple traffic classes
7635 * @netdev: net device to configure
7636 * @type_data: tc offload data
7638 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7640 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7641 struct i40e_netdev_priv *np = netdev_priv(netdev);
7642 struct i40e_vsi *vsi = np->vsi;
7643 struct i40e_pf *pf = vsi->back;
7644 u8 enabled_tc = 0, num_tc, hw;
7645 bool need_reset = false;
7646 int old_queue_pairs;
7651 old_queue_pairs = vsi->num_queue_pairs;
7652 num_tc = mqprio_qopt->qopt.num_tc;
7653 hw = mqprio_qopt->qopt.hw;
7654 mode = mqprio_qopt->mode;
7656 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7657 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7661 /* Check if MFP enabled */
7662 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7664 "Configuring TC not supported in MFP mode\n");
7668 case TC_MQPRIO_MODE_DCB:
7669 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7671 /* Check if DCB enabled to continue */
7672 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7674 "DCB is not enabled for adapter\n");
7678 /* Check whether tc count is within enabled limit */
7679 if (num_tc > i40e_pf_get_num_tc(pf)) {
7681 "TC count greater than enabled on link for adapter\n");
7685 case TC_MQPRIO_MODE_CHANNEL:
7686 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7688 "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7691 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7693 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7696 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7697 sizeof(*mqprio_qopt));
7698 pf->flags |= I40E_FLAG_TC_MQPRIO;
7699 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7706 /* Generate TC map for number of tc requested */
7707 for (i = 0; i < num_tc; i++)
7708 enabled_tc |= BIT(i);
7710 /* Requesting same TC configuration as already enabled */
7711 if (enabled_tc == vsi->tc_config.enabled_tc &&
7712 mode != TC_MQPRIO_MODE_CHANNEL)
7715 /* Quiesce VSI queues */
7716 i40e_quiesce_vsi(vsi);
7718 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7719 i40e_remove_queue_channels(vsi);
7721 /* Configure VSI for enabled TCs */
7722 ret = i40e_vsi_config_tc(vsi, enabled_tc);
7724 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7728 } else if (enabled_tc &&
7729 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
7731 "Failed to create channel. Override queues (%u) not power of 2\n",
7732 vsi->tc_config.tc_info[0].qcount);
7738 dev_info(&vsi->back->pdev->dev,
7739 "Setup channel (id:%u) utilizing num_queues %d\n",
7740 vsi->seid, vsi->tc_config.tc_info[0].qcount);
7742 if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7743 if (vsi->mqprio_qopt.max_rate[0]) {
7744 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
7745 vsi->mqprio_qopt.max_rate[0]);
7747 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
7749 u64 credits = max_tx_rate;
7751 do_div(credits, I40E_BW_CREDIT_DIVISOR);
7752 dev_dbg(&vsi->back->pdev->dev,
7753 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
7762 ret = i40e_configure_queue_channels(vsi);
7764 vsi->num_queue_pairs = old_queue_pairs;
7766 "Failed configuring queue channels\n");
7773 /* Reset the configuration data to defaults, only TC0 is enabled */
7775 i40e_vsi_set_default_tc_config(vsi);
7780 i40e_unquiesce_vsi(vsi);
7785 * i40e_set_cld_element - sets cloud filter element data
7786 * @filter: cloud filter rule
7787 * @cld: ptr to cloud filter element data
7789 * This is helper function to copy data into cloud filter element
7792 i40e_set_cld_element(struct i40e_cloud_filter *filter,
7793 struct i40e_aqc_cloud_filters_element_data *cld)
7798 memset(cld, 0, sizeof(*cld));
7799 ether_addr_copy(cld->outer_mac, filter->dst_mac);
7800 ether_addr_copy(cld->inner_mac, filter->src_mac);
7802 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
7805 if (filter->n_proto == ETH_P_IPV6) {
7806 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)
7807 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
7808 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
7810 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
7813 ipa = be32_to_cpu(filter->dst_ipv4);
7815 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
7818 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
7820 /* tenant_id is not supported by FW now, once the support is enabled
7821 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
7823 if (filter->tenant_id)
7828 * i40e_add_del_cloud_filter - Add/del cloud filter
7829 * @vsi: pointer to VSI
7830 * @filter: cloud filter rule
7831 * @add: if true, add, if false, delete
7833 * Add or delete a cloud filter for a specific flow spec.
7834 * Returns 0 if the filter were successfully added.
7836 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
7837 struct i40e_cloud_filter *filter, bool add)
7839 struct i40e_aqc_cloud_filters_element_data cld_filter;
7840 struct i40e_pf *pf = vsi->back;
7842 static const u16 flag_table[128] = {
7843 [I40E_CLOUD_FILTER_FLAGS_OMAC] =
7844 I40E_AQC_ADD_CLOUD_FILTER_OMAC,
7845 [I40E_CLOUD_FILTER_FLAGS_IMAC] =
7846 I40E_AQC_ADD_CLOUD_FILTER_IMAC,
7847 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =
7848 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
7849 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
7850 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
7851 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
7852 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
7853 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
7854 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
7855 [I40E_CLOUD_FILTER_FLAGS_IIP] =
7856 I40E_AQC_ADD_CLOUD_FILTER_IIP,
7859 if (filter->flags >= ARRAY_SIZE(flag_table))
7860 return I40E_ERR_CONFIG;
7862 memset(&cld_filter, 0, sizeof(cld_filter));
7864 /* copy element needed to add cloud filter from filter */
7865 i40e_set_cld_element(filter, &cld_filter);
7867 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
7868 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
7869 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
7871 if (filter->n_proto == ETH_P_IPV6)
7872 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7873 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7875 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7876 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7879 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
7882 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
7885 dev_dbg(&pf->pdev->dev,
7886 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
7887 add ? "add" : "delete", filter->dst_port, ret,
7888 pf->hw.aq.asq_last_status);
7890 dev_info(&pf->pdev->dev,
7891 "%s cloud filter for VSI: %d\n",
7892 add ? "Added" : "Deleted", filter->seid);
7897 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
7898 * @vsi: pointer to VSI
7899 * @filter: cloud filter rule
7900 * @add: if true, add, if false, delete
7902 * Add or delete a cloud filter for a specific flow spec using big buffer.
7903 * Returns 0 if the filter were successfully added.
7905 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
7906 struct i40e_cloud_filter *filter,
7909 struct i40e_aqc_cloud_filters_element_bb cld_filter;
7910 struct i40e_pf *pf = vsi->back;
7913 /* Both (src/dst) valid mac_addr are not supported */
7914 if ((is_valid_ether_addr(filter->dst_mac) &&
7915 is_valid_ether_addr(filter->src_mac)) ||
7916 (is_multicast_ether_addr(filter->dst_mac) &&
7917 is_multicast_ether_addr(filter->src_mac)))
7920 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
7921 * ports are not supported via big buffer now.
7923 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
7926 /* adding filter using src_port/src_ip is not supported at this stage */
7927 if (filter->src_port ||
7928 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
7929 !ipv6_addr_any(&filter->ip.v6.src_ip6))
7932 memset(&cld_filter, 0, sizeof(cld_filter));
7934 /* copy element needed to add cloud filter from filter */
7935 i40e_set_cld_element(filter, &cld_filter.element);
7937 if (is_valid_ether_addr(filter->dst_mac) ||
7938 is_valid_ether_addr(filter->src_mac) ||
7939 is_multicast_ether_addr(filter->dst_mac) ||
7940 is_multicast_ether_addr(filter->src_mac)) {
7941 /* MAC + IP : unsupported mode */
7942 if (filter->dst_ipv4)
7945 /* since we validated that L4 port must be valid before
7946 * we get here, start with respective "flags" value
7947 * and update if vlan is present or not
7949 cld_filter.element.flags =
7950 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
7952 if (filter->vlan_id) {
7953 cld_filter.element.flags =
7954 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
7957 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
7958 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
7959 cld_filter.element.flags =
7960 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
7961 if (filter->n_proto == ETH_P_IPV6)
7962 cld_filter.element.flags |=
7963 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7965 cld_filter.element.flags |=
7966 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7968 dev_err(&pf->pdev->dev,
7969 "either mac or ip has to be valid for cloud filter\n");
7973 /* Now copy L4 port in Byte 6..7 in general fields */
7974 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
7975 be16_to_cpu(filter->dst_port);
7978 /* Validate current device switch mode, change if necessary */
7979 ret = i40e_validate_and_set_switch_mode(vsi);
7981 dev_err(&pf->pdev->dev,
7982 "failed to set switch mode, ret %d\n",
7987 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
7990 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
7995 dev_dbg(&pf->pdev->dev,
7996 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
7997 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
7999 dev_info(&pf->pdev->dev,
8000 "%s cloud filter for VSI: %d, L4 port: %d\n",
8001 add ? "add" : "delete", filter->seid,
8002 ntohs(filter->dst_port));
8007 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8008 * @vsi: Pointer to VSI
8009 * @f: Pointer to struct flow_cls_offload
8010 * @filter: Pointer to cloud filter structure
8013 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8014 struct flow_cls_offload *f,
8015 struct i40e_cloud_filter *filter)
8017 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8018 struct flow_dissector *dissector = rule->match.dissector;
8019 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8020 struct i40e_pf *pf = vsi->back;
8023 if (dissector->used_keys &
8024 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8025 BIT(FLOW_DISSECTOR_KEY_BASIC) |
8026 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8027 BIT(FLOW_DISSECTOR_KEY_VLAN) |
8028 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8029 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8030 BIT(FLOW_DISSECTOR_KEY_PORTS) |
8031 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8032 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8033 dissector->used_keys);
8037 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8038 struct flow_match_enc_keyid match;
8040 flow_rule_match_enc_keyid(rule, &match);
8041 if (match.mask->keyid != 0)
8042 field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8044 filter->tenant_id = be32_to_cpu(match.key->keyid);
8047 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8048 struct flow_match_basic match;
8050 flow_rule_match_basic(rule, &match);
8051 n_proto_key = ntohs(match.key->n_proto);
8052 n_proto_mask = ntohs(match.mask->n_proto);
8054 if (n_proto_key == ETH_P_ALL) {
8058 filter->n_proto = n_proto_key & n_proto_mask;
8059 filter->ip_proto = match.key->ip_proto;
8062 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8063 struct flow_match_eth_addrs match;
8065 flow_rule_match_eth_addrs(rule, &match);
8067 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8068 if (!is_zero_ether_addr(match.mask->dst)) {
8069 if (is_broadcast_ether_addr(match.mask->dst)) {
8070 field_flags |= I40E_CLOUD_FIELD_OMAC;
8072 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8074 return I40E_ERR_CONFIG;
8078 if (!is_zero_ether_addr(match.mask->src)) {
8079 if (is_broadcast_ether_addr(match.mask->src)) {
8080 field_flags |= I40E_CLOUD_FIELD_IMAC;
8082 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8084 return I40E_ERR_CONFIG;
8087 ether_addr_copy(filter->dst_mac, match.key->dst);
8088 ether_addr_copy(filter->src_mac, match.key->src);
8091 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8092 struct flow_match_vlan match;
8094 flow_rule_match_vlan(rule, &match);
8095 if (match.mask->vlan_id) {
8096 if (match.mask->vlan_id == VLAN_VID_MASK) {
8097 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8100 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8101 match.mask->vlan_id);
8102 return I40E_ERR_CONFIG;
8106 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8109 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8110 struct flow_match_control match;
8112 flow_rule_match_control(rule, &match);
8113 addr_type = match.key->addr_type;
8116 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8117 struct flow_match_ipv4_addrs match;
8119 flow_rule_match_ipv4_addrs(rule, &match);
8120 if (match.mask->dst) {
8121 if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8122 field_flags |= I40E_CLOUD_FIELD_IIP;
8124 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8126 return I40E_ERR_CONFIG;
8130 if (match.mask->src) {
8131 if (match.mask->src == cpu_to_be32(0xffffffff)) {
8132 field_flags |= I40E_CLOUD_FIELD_IIP;
8134 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8136 return I40E_ERR_CONFIG;
8140 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8141 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8142 return I40E_ERR_CONFIG;
8144 filter->dst_ipv4 = match.key->dst;
8145 filter->src_ipv4 = match.key->src;
8148 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8149 struct flow_match_ipv6_addrs match;
8151 flow_rule_match_ipv6_addrs(rule, &match);
8153 /* src and dest IPV6 address should not be LOOPBACK
8154 * (0:0:0:0:0:0:0:1), which can be represented as ::1
8156 if (ipv6_addr_loopback(&match.key->dst) ||
8157 ipv6_addr_loopback(&match.key->src)) {
8158 dev_err(&pf->pdev->dev,
8159 "Bad ipv6, addr is LOOPBACK\n");
8160 return I40E_ERR_CONFIG;
8162 if (!ipv6_addr_any(&match.mask->dst) ||
8163 !ipv6_addr_any(&match.mask->src))
8164 field_flags |= I40E_CLOUD_FIELD_IIP;
8166 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8167 sizeof(filter->src_ipv6));
8168 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8169 sizeof(filter->dst_ipv6));
8172 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8173 struct flow_match_ports match;
8175 flow_rule_match_ports(rule, &match);
8176 if (match.mask->src) {
8177 if (match.mask->src == cpu_to_be16(0xffff)) {
8178 field_flags |= I40E_CLOUD_FIELD_IIP;
8180 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8181 be16_to_cpu(match.mask->src));
8182 return I40E_ERR_CONFIG;
8186 if (match.mask->dst) {
8187 if (match.mask->dst == cpu_to_be16(0xffff)) {
8188 field_flags |= I40E_CLOUD_FIELD_IIP;
8190 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8191 be16_to_cpu(match.mask->dst));
8192 return I40E_ERR_CONFIG;
8196 filter->dst_port = match.key->dst;
8197 filter->src_port = match.key->src;
8199 switch (filter->ip_proto) {
8204 dev_err(&pf->pdev->dev,
8205 "Only UDP and TCP transport are supported\n");
8209 filter->flags = field_flags;
8214 * i40e_handle_tclass: Forward to a traffic class on the device
8215 * @vsi: Pointer to VSI
8216 * @tc: traffic class index on the device
8217 * @filter: Pointer to cloud filter structure
8220 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8221 struct i40e_cloud_filter *filter)
8223 struct i40e_channel *ch, *ch_tmp;
8225 /* direct to a traffic class on the same device */
8227 filter->seid = vsi->seid;
8229 } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8230 if (!filter->dst_port) {
8231 dev_err(&vsi->back->pdev->dev,
8232 "Specify destination port to direct to traffic class that is not default\n");
8235 if (list_empty(&vsi->ch_list))
8237 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8239 if (ch->seid == vsi->tc_seid_map[tc])
8240 filter->seid = ch->seid;
8244 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8249 * i40e_configure_clsflower - Configure tc flower filters
8250 * @vsi: Pointer to VSI
8251 * @cls_flower: Pointer to struct flow_cls_offload
8254 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8255 struct flow_cls_offload *cls_flower)
8257 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8258 struct i40e_cloud_filter *filter = NULL;
8259 struct i40e_pf *pf = vsi->back;
8263 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8268 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8272 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8273 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8276 if (pf->fdir_pf_active_filters ||
8277 (!hlist_empty(&pf->fdir_filter_list))) {
8278 dev_err(&vsi->back->pdev->dev,
8279 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8283 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8284 dev_err(&vsi->back->pdev->dev,
8285 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8286 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8287 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8290 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8294 filter->cookie = cls_flower->cookie;
8296 err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8300 err = i40e_handle_tclass(vsi, tc, filter);
8304 /* Add cloud filter */
8305 if (filter->dst_port)
8306 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8308 err = i40e_add_del_cloud_filter(vsi, filter, true);
8311 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8316 /* add filter to the ordered list */
8317 INIT_HLIST_NODE(&filter->cloud_node);
8319 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8321 pf->num_cloud_filters++;
8330 * i40e_find_cloud_filter - Find the could filter in the list
8331 * @vsi: Pointer to VSI
8332 * @cookie: filter specific cookie
8335 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8336 unsigned long *cookie)
8338 struct i40e_cloud_filter *filter = NULL;
8339 struct hlist_node *node2;
8341 hlist_for_each_entry_safe(filter, node2,
8342 &vsi->back->cloud_filter_list, cloud_node)
8343 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8349 * i40e_delete_clsflower - Remove tc flower filters
8350 * @vsi: Pointer to VSI
8351 * @cls_flower: Pointer to struct flow_cls_offload
8354 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8355 struct flow_cls_offload *cls_flower)
8357 struct i40e_cloud_filter *filter = NULL;
8358 struct i40e_pf *pf = vsi->back;
8361 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8366 hash_del(&filter->cloud_node);
8368 if (filter->dst_port)
8369 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8371 err = i40e_add_del_cloud_filter(vsi, filter, false);
8375 dev_err(&pf->pdev->dev,
8376 "Failed to delete cloud filter, err %s\n",
8377 i40e_stat_str(&pf->hw, err));
8378 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8381 pf->num_cloud_filters--;
8382 if (!pf->num_cloud_filters)
8383 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8384 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8385 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8386 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8387 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8393 * i40e_setup_tc_cls_flower - flower classifier offloads
8394 * @np: net device to configure
8395 * @cls_flower: offload data
8397 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8398 struct flow_cls_offload *cls_flower)
8400 struct i40e_vsi *vsi = np->vsi;
8402 switch (cls_flower->command) {
8403 case FLOW_CLS_REPLACE:
8404 return i40e_configure_clsflower(vsi, cls_flower);
8405 case FLOW_CLS_DESTROY:
8406 return i40e_delete_clsflower(vsi, cls_flower);
8407 case FLOW_CLS_STATS:
8414 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8417 struct i40e_netdev_priv *np = cb_priv;
8419 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8423 case TC_SETUP_CLSFLOWER:
8424 return i40e_setup_tc_cls_flower(np, type_data);
8431 static LIST_HEAD(i40e_block_cb_list);
8433 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8436 struct i40e_netdev_priv *np = netdev_priv(netdev);
8439 case TC_SETUP_QDISC_MQPRIO:
8440 return i40e_setup_tc(netdev, type_data);
8441 case TC_SETUP_BLOCK:
8442 return flow_block_cb_setup_simple(type_data,
8443 &i40e_block_cb_list,
8444 i40e_setup_tc_block_cb,
8452 * i40e_open - Called when a network interface is made active
8453 * @netdev: network interface device structure
8455 * The open entry point is called when a network interface is made
8456 * active by the system (IFF_UP). At this point all resources needed
8457 * for transmit and receive operations are allocated, the interrupt
8458 * handler is registered with the OS, the netdev watchdog subtask is
8459 * enabled, and the stack is notified that the interface is ready.
8461 * Returns 0 on success, negative value on failure
8463 int i40e_open(struct net_device *netdev)
8465 struct i40e_netdev_priv *np = netdev_priv(netdev);
8466 struct i40e_vsi *vsi = np->vsi;
8467 struct i40e_pf *pf = vsi->back;
8470 /* disallow open during test or if eeprom is broken */
8471 if (test_bit(__I40E_TESTING, pf->state) ||
8472 test_bit(__I40E_BAD_EEPROM, pf->state))
8475 netif_carrier_off(netdev);
8477 if (i40e_force_link_state(pf, true))
8480 err = i40e_vsi_open(vsi);
8484 /* configure global TSO hardware offload settings */
8485 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8486 TCP_FLAG_FIN) >> 16);
8487 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8489 TCP_FLAG_CWR) >> 16);
8490 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8492 udp_tunnel_get_rx_info(netdev);
8498 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
8499 * @vsi: vsi structure
8501 * This updates netdev's number of tx/rx queues
8503 * Returns status of setting tx/rx queues
8505 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
8509 ret = netif_set_real_num_rx_queues(vsi->netdev,
8510 vsi->num_queue_pairs);
8514 return netif_set_real_num_tx_queues(vsi->netdev,
8515 vsi->num_queue_pairs);
8520 * @vsi: the VSI to open
8522 * Finish initialization of the VSI.
8524 * Returns 0 on success, negative value on failure
8526 * Note: expects to be called while under rtnl_lock()
8528 int i40e_vsi_open(struct i40e_vsi *vsi)
8530 struct i40e_pf *pf = vsi->back;
8531 char int_name[I40E_INT_NAME_STR_LEN];
8534 /* allocate descriptors */
8535 err = i40e_vsi_setup_tx_resources(vsi);
8538 err = i40e_vsi_setup_rx_resources(vsi);
8542 err = i40e_vsi_configure(vsi);
8547 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8548 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8549 err = i40e_vsi_request_irq(vsi, int_name);
8553 /* Notify the stack of the actual queue counts. */
8554 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
8556 goto err_set_queues;
8558 } else if (vsi->type == I40E_VSI_FDIR) {
8559 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8560 dev_driver_string(&pf->pdev->dev),
8561 dev_name(&pf->pdev->dev));
8562 err = i40e_vsi_request_irq(vsi, int_name);
8571 err = i40e_up_complete(vsi);
8573 goto err_up_complete;
8580 i40e_vsi_free_irq(vsi);
8582 i40e_vsi_free_rx_resources(vsi);
8584 i40e_vsi_free_tx_resources(vsi);
8585 if (vsi == pf->vsi[pf->lan_vsi])
8586 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8592 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
8593 * @pf: Pointer to PF
8595 * This function destroys the hlist where all the Flow Director
8596 * filters were saved.
8598 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8600 struct i40e_fdir_filter *filter;
8601 struct i40e_flex_pit *pit_entry, *tmp;
8602 struct hlist_node *node2;
8604 hlist_for_each_entry_safe(filter, node2,
8605 &pf->fdir_filter_list, fdir_node) {
8606 hlist_del(&filter->fdir_node);
8610 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8611 list_del(&pit_entry->list);
8614 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8616 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8617 list_del(&pit_entry->list);
8620 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8622 pf->fdir_pf_active_filters = 0;
8623 pf->fd_tcp4_filter_cnt = 0;
8624 pf->fd_udp4_filter_cnt = 0;
8625 pf->fd_sctp4_filter_cnt = 0;
8626 pf->fd_ip4_filter_cnt = 0;
8628 /* Reprogram the default input set for TCP/IPv4 */
8629 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8630 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8631 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8633 /* Reprogram the default input set for UDP/IPv4 */
8634 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8635 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8636 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8638 /* Reprogram the default input set for SCTP/IPv4 */
8639 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8640 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8641 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8643 /* Reprogram the default input set for Other/IPv4 */
8644 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8645 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8647 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8648 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8652 * i40e_cloud_filter_exit - Cleans up the cloud filters
8653 * @pf: Pointer to PF
8655 * This function destroys the hlist where all the cloud filters
8658 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8660 struct i40e_cloud_filter *cfilter;
8661 struct hlist_node *node;
8663 hlist_for_each_entry_safe(cfilter, node,
8664 &pf->cloud_filter_list, cloud_node) {
8665 hlist_del(&cfilter->cloud_node);
8668 pf->num_cloud_filters = 0;
8670 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8671 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8672 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8673 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8674 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8679 * i40e_close - Disables a network interface
8680 * @netdev: network interface device structure
8682 * The close entry point is called when an interface is de-activated
8683 * by the OS. The hardware is still under the driver's control, but
8684 * this netdev interface is disabled.
8686 * Returns 0, this is not allowed to fail
8688 int i40e_close(struct net_device *netdev)
8690 struct i40e_netdev_priv *np = netdev_priv(netdev);
8691 struct i40e_vsi *vsi = np->vsi;
8693 i40e_vsi_close(vsi);
8699 * i40e_do_reset - Start a PF or Core Reset sequence
8700 * @pf: board private structure
8701 * @reset_flags: which reset is requested
8702 * @lock_acquired: indicates whether or not the lock has been acquired
8703 * before this function was called.
8705 * The essential difference in resets is that the PF Reset
8706 * doesn't clear the packet buffers, doesn't reset the PE
8707 * firmware, and doesn't bother the other PFs on the chip.
8709 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8713 /* do the biggest reset indicated */
8714 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8716 /* Request a Global Reset
8718 * This will start the chip's countdown to the actual full
8719 * chip reset event, and a warning interrupt to be sent
8720 * to all PFs, including the requestor. Our handler
8721 * for the warning interrupt will deal with the shutdown
8722 * and recovery of the switch setup.
8724 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8725 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8726 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8727 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8729 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8731 /* Request a Core Reset
8733 * Same as Global Reset, except does *not* include the MAC/PHY
8735 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8736 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8737 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8738 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8739 i40e_flush(&pf->hw);
8741 } else if (reset_flags & I40E_PF_RESET_FLAG) {
8743 /* Request a PF Reset
8745 * Resets only the PF-specific registers
8747 * This goes directly to the tear-down and rebuild of
8748 * the switch, since we need to do all the recovery as
8749 * for the Core Reset.
8751 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8752 i40e_handle_reset_warning(pf, lock_acquired);
8754 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
8755 /* Request a PF Reset
8757 * Resets PF and reinitializes PFs VSI.
8759 i40e_prep_for_reset(pf, lock_acquired);
8760 i40e_reset_and_rebuild(pf, true, lock_acquired);
8761 dev_info(&pf->pdev->dev,
8762 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
8763 "FW LLDP is disabled\n" :
8764 "FW LLDP is enabled\n");
8766 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8769 /* Find the VSI(s) that requested a re-init */
8770 dev_info(&pf->pdev->dev,
8771 "VSI reinit requested\n");
8772 for (v = 0; v < pf->num_alloc_vsi; v++) {
8773 struct i40e_vsi *vsi = pf->vsi[v];
8776 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
8778 i40e_vsi_reinit_locked(pf->vsi[v]);
8780 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
8783 /* Find the VSI(s) that needs to be brought down */
8784 dev_info(&pf->pdev->dev, "VSI down requested\n");
8785 for (v = 0; v < pf->num_alloc_vsi; v++) {
8786 struct i40e_vsi *vsi = pf->vsi[v];
8789 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
8791 set_bit(__I40E_VSI_DOWN, vsi->state);
8796 dev_info(&pf->pdev->dev,
8797 "bad reset request 0x%08x\n", reset_flags);
8801 #ifdef CONFIG_I40E_DCB
8803 * i40e_dcb_need_reconfig - Check if DCB needs reconfig
8804 * @pf: board private structure
8805 * @old_cfg: current DCB config
8806 * @new_cfg: new DCB config
8808 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
8809 struct i40e_dcbx_config *old_cfg,
8810 struct i40e_dcbx_config *new_cfg)
8812 bool need_reconfig = false;
8814 /* Check if ETS configuration has changed */
8815 if (memcmp(&new_cfg->etscfg,
8817 sizeof(new_cfg->etscfg))) {
8818 /* If Priority Table has changed reconfig is needed */
8819 if (memcmp(&new_cfg->etscfg.prioritytable,
8820 &old_cfg->etscfg.prioritytable,
8821 sizeof(new_cfg->etscfg.prioritytable))) {
8822 need_reconfig = true;
8823 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
8826 if (memcmp(&new_cfg->etscfg.tcbwtable,
8827 &old_cfg->etscfg.tcbwtable,
8828 sizeof(new_cfg->etscfg.tcbwtable)))
8829 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
8831 if (memcmp(&new_cfg->etscfg.tsatable,
8832 &old_cfg->etscfg.tsatable,
8833 sizeof(new_cfg->etscfg.tsatable)))
8834 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
8837 /* Check if PFC configuration has changed */
8838 if (memcmp(&new_cfg->pfc,
8840 sizeof(new_cfg->pfc))) {
8841 need_reconfig = true;
8842 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
8845 /* Check if APP Table has changed */
8846 if (memcmp(&new_cfg->app,
8848 sizeof(new_cfg->app))) {
8849 need_reconfig = true;
8850 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
8853 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
8854 return need_reconfig;
8858 * i40e_handle_lldp_event - Handle LLDP Change MIB event
8859 * @pf: board private structure
8860 * @e: event info posted on ARQ
8862 static int i40e_handle_lldp_event(struct i40e_pf *pf,
8863 struct i40e_arq_event_info *e)
8865 struct i40e_aqc_lldp_get_mib *mib =
8866 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
8867 struct i40e_hw *hw = &pf->hw;
8868 struct i40e_dcbx_config tmp_dcbx_cfg;
8869 bool need_reconfig = false;
8873 /* Not DCB capable or capability disabled */
8874 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
8877 /* Ignore if event is not for Nearest Bridge */
8878 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
8879 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
8880 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
8881 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
8884 /* Check MIB Type and return if event for Remote MIB update */
8885 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
8886 dev_dbg(&pf->pdev->dev,
8887 "LLDP event mib type %s\n", type ? "remote" : "local");
8888 if (type == I40E_AQ_LLDP_MIB_REMOTE) {
8889 /* Update the remote cached instance and return */
8890 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
8891 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
8892 &hw->remote_dcbx_config);
8896 /* Store the old configuration */
8897 tmp_dcbx_cfg = hw->local_dcbx_config;
8899 /* Reset the old DCBx configuration data */
8900 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
8901 /* Get updated DCBX data from firmware */
8902 ret = i40e_get_dcb_config(&pf->hw);
8904 dev_info(&pf->pdev->dev,
8905 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
8906 i40e_stat_str(&pf->hw, ret),
8907 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8911 /* No change detected in DCBX configs */
8912 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
8913 sizeof(tmp_dcbx_cfg))) {
8914 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
8918 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
8919 &hw->local_dcbx_config);
8921 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
8926 /* Enable DCB tagging only when more than one TC */
8927 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
8928 pf->flags |= I40E_FLAG_DCB_ENABLED;
8930 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8932 set_bit(__I40E_PORT_SUSPENDED, pf->state);
8933 /* Reconfiguration needed quiesce all VSIs */
8934 i40e_pf_quiesce_all_vsi(pf);
8936 /* Changes in configuration update VEB/VSI */
8937 i40e_dcb_reconfigure(pf);
8939 ret = i40e_resume_port_tx(pf);
8941 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
8942 /* In case of error no point in resuming VSIs */
8946 /* Wait for the PF's queues to be disabled */
8947 ret = i40e_pf_wait_queues_disabled(pf);
8949 /* Schedule PF reset to recover */
8950 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8951 i40e_service_event_schedule(pf);
8953 i40e_pf_unquiesce_all_vsi(pf);
8954 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
8955 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
8961 #endif /* CONFIG_I40E_DCB */
8964 * i40e_do_reset_safe - Protected reset path for userland calls.
8965 * @pf: board private structure
8966 * @reset_flags: which reset is requested
8969 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
8972 i40e_do_reset(pf, reset_flags, true);
8977 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
8978 * @pf: board private structure
8979 * @e: event info posted on ARQ
8981 * Handler for LAN Queue Overflow Event generated by the firmware for PF
8984 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
8985 struct i40e_arq_event_info *e)
8987 struct i40e_aqc_lan_overflow *data =
8988 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
8989 u32 queue = le32_to_cpu(data->prtdcb_rupto);
8990 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
8991 struct i40e_hw *hw = &pf->hw;
8995 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
8998 /* Queue belongs to VF, find the VF and issue VF reset */
8999 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9000 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9001 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9002 >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9003 vf_id -= hw->func_caps.vf_base_id;
9004 vf = &pf->vf[vf_id];
9005 i40e_vc_notify_vf_reset(vf);
9006 /* Allow VF to process pending reset notification */
9008 i40e_reset_vf(vf, false);
9013 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9014 * @pf: board private structure
9016 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9020 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9021 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9026 * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9027 * @pf: board private structure
9029 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9033 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9034 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9035 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9036 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9041 * i40e_get_global_fd_count - Get total FD filters programmed on device
9042 * @pf: board private structure
9044 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9048 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9049 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9050 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9051 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9056 * i40e_reenable_fdir_sb - Restore FDir SB capability
9057 * @pf: board private structure
9059 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9061 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9062 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9063 (I40E_DEBUG_FD & pf->hw.debug_mask))
9064 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9068 * i40e_reenable_fdir_atr - Restore FDir ATR capability
9069 * @pf: board private structure
9071 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9073 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9074 /* ATR uses the same filtering logic as SB rules. It only
9075 * functions properly if the input set mask is at the default
9076 * settings. It is safe to restore the default input set
9077 * because there are no active TCPv4 filter rules.
9079 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9080 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9081 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9083 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9084 (I40E_DEBUG_FD & pf->hw.debug_mask))
9085 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9090 * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9091 * @pf: board private structure
9092 * @filter: FDir filter to remove
9094 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9095 struct i40e_fdir_filter *filter)
9097 /* Update counters */
9098 pf->fdir_pf_active_filters--;
9101 switch (filter->flow_type) {
9103 pf->fd_tcp4_filter_cnt--;
9106 pf->fd_udp4_filter_cnt--;
9109 pf->fd_sctp4_filter_cnt--;
9112 switch (filter->ip4_proto) {
9114 pf->fd_tcp4_filter_cnt--;
9117 pf->fd_udp4_filter_cnt--;
9120 pf->fd_sctp4_filter_cnt--;
9123 pf->fd_ip4_filter_cnt--;
9129 /* Remove the filter from the list and free memory */
9130 hlist_del(&filter->fdir_node);
9135 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9136 * @pf: board private structure
9138 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9140 struct i40e_fdir_filter *filter;
9141 u32 fcnt_prog, fcnt_avail;
9142 struct hlist_node *node;
9144 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9147 /* Check if we have enough room to re-enable FDir SB capability. */
9148 fcnt_prog = i40e_get_global_fd_count(pf);
9149 fcnt_avail = pf->fdir_pf_filter_count;
9150 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9151 (pf->fd_add_err == 0) ||
9152 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9153 i40e_reenable_fdir_sb(pf);
9155 /* We should wait for even more space before re-enabling ATR.
9156 * Additionally, we cannot enable ATR as long as we still have TCP SB
9159 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9160 (pf->fd_tcp4_filter_cnt == 0))
9161 i40e_reenable_fdir_atr(pf);
9163 /* if hw had a problem adding a filter, delete it */
9164 if (pf->fd_inv > 0) {
9165 hlist_for_each_entry_safe(filter, node,
9166 &pf->fdir_filter_list, fdir_node)
9167 if (filter->fd_id == pf->fd_inv)
9168 i40e_delete_invalid_filter(pf, filter);
9172 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9173 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9175 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9176 * @pf: board private structure
9178 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9180 unsigned long min_flush_time;
9181 int flush_wait_retry = 50;
9182 bool disable_atr = false;
9186 if (!time_after(jiffies, pf->fd_flush_timestamp +
9187 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9190 /* If the flush is happening too quick and we have mostly SB rules we
9191 * should not re-enable ATR for some time.
9193 min_flush_time = pf->fd_flush_timestamp +
9194 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9195 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9197 if (!(time_after(jiffies, min_flush_time)) &&
9198 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9199 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9200 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9204 pf->fd_flush_timestamp = jiffies;
9205 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9206 /* flush all filters */
9207 wr32(&pf->hw, I40E_PFQF_CTL_1,
9208 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9209 i40e_flush(&pf->hw);
9213 /* Check FD flush status every 5-6msec */
9214 usleep_range(5000, 6000);
9215 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9216 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9218 } while (flush_wait_retry--);
9219 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9220 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9222 /* replay sideband filters */
9223 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9224 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9225 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9226 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9227 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9228 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9233 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
9234 * @pf: board private structure
9236 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9238 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9242 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9243 * @pf: board private structure
9245 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9248 /* if interface is down do nothing */
9249 if (test_bit(__I40E_DOWN, pf->state))
9252 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9253 i40e_fdir_flush_and_replay(pf);
9255 i40e_fdir_check_and_reenable(pf);
9260 * i40e_vsi_link_event - notify VSI of a link event
9261 * @vsi: vsi to be notified
9262 * @link_up: link up or down
9264 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9266 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9269 switch (vsi->type) {
9271 if (!vsi->netdev || !vsi->netdev_registered)
9275 netif_carrier_on(vsi->netdev);
9276 netif_tx_wake_all_queues(vsi->netdev);
9278 netif_carrier_off(vsi->netdev);
9279 netif_tx_stop_all_queues(vsi->netdev);
9283 case I40E_VSI_SRIOV:
9284 case I40E_VSI_VMDQ2:
9286 case I40E_VSI_IWARP:
9287 case I40E_VSI_MIRROR:
9289 /* there is no notification for other VSIs */
9295 * i40e_veb_link_event - notify elements on the veb of a link event
9296 * @veb: veb to be notified
9297 * @link_up: link up or down
9299 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9304 if (!veb || !veb->pf)
9308 /* depth first... */
9309 for (i = 0; i < I40E_MAX_VEB; i++)
9310 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9311 i40e_veb_link_event(pf->veb[i], link_up);
9313 /* ... now the local VSIs */
9314 for (i = 0; i < pf->num_alloc_vsi; i++)
9315 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9316 i40e_vsi_link_event(pf->vsi[i], link_up);
9320 * i40e_link_event - Update netif_carrier status
9321 * @pf: board private structure
9323 static void i40e_link_event(struct i40e_pf *pf)
9325 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9326 u8 new_link_speed, old_link_speed;
9328 bool new_link, old_link;
9330 /* set this to force the get_link_status call to refresh state */
9331 pf->hw.phy.get_link_info = true;
9332 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9333 status = i40e_get_link_status(&pf->hw, &new_link);
9335 /* On success, disable temp link polling */
9336 if (status == I40E_SUCCESS) {
9337 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9339 /* Enable link polling temporarily until i40e_get_link_status
9340 * returns I40E_SUCCESS
9342 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9343 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9348 old_link_speed = pf->hw.phy.link_info_old.link_speed;
9349 new_link_speed = pf->hw.phy.link_info.link_speed;
9351 if (new_link == old_link &&
9352 new_link_speed == old_link_speed &&
9353 (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9354 new_link == netif_carrier_ok(vsi->netdev)))
9357 i40e_print_link_message(vsi, new_link);
9359 /* Notify the base of the switch tree connected to
9360 * the link. Floating VEBs are not notified.
9362 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9363 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9365 i40e_vsi_link_event(vsi, new_link);
9368 i40e_vc_notify_link_state(pf);
9370 if (pf->flags & I40E_FLAG_PTP)
9371 i40e_ptp_set_increment(pf);
9375 * i40e_watchdog_subtask - periodic checks not using event driven response
9376 * @pf: board private structure
9378 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9382 /* if interface is down do nothing */
9383 if (test_bit(__I40E_DOWN, pf->state) ||
9384 test_bit(__I40E_CONFIG_BUSY, pf->state))
9387 /* make sure we don't do these things too often */
9388 if (time_before(jiffies, (pf->service_timer_previous +
9389 pf->service_timer_period)))
9391 pf->service_timer_previous = jiffies;
9393 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9394 test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9395 i40e_link_event(pf);
9397 /* Update the stats for active netdevs so the network stack
9398 * can look at updated numbers whenever it cares to
9400 for (i = 0; i < pf->num_alloc_vsi; i++)
9401 if (pf->vsi[i] && pf->vsi[i]->netdev)
9402 i40e_update_stats(pf->vsi[i]);
9404 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9405 /* Update the stats for the active switching components */
9406 for (i = 0; i < I40E_MAX_VEB; i++)
9408 i40e_update_veb_stats(pf->veb[i]);
9411 i40e_ptp_rx_hang(pf);
9412 i40e_ptp_tx_hang(pf);
9416 * i40e_reset_subtask - Set up for resetting the device and driver
9417 * @pf: board private structure
9419 static void i40e_reset_subtask(struct i40e_pf *pf)
9421 u32 reset_flags = 0;
9423 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9424 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9425 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9427 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9428 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9429 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9431 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9432 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9433 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9435 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9436 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9437 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9439 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9440 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9441 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9444 /* If there's a recovery already waiting, it takes
9445 * precedence before starting a new reset sequence.
9447 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9448 i40e_prep_for_reset(pf, false);
9450 i40e_rebuild(pf, false, false);
9453 /* If we're already down or resetting, just bail */
9455 !test_bit(__I40E_DOWN, pf->state) &&
9456 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9457 i40e_do_reset(pf, reset_flags, false);
9462 * i40e_handle_link_event - Handle link event
9463 * @pf: board private structure
9464 * @e: event info posted on ARQ
9466 static void i40e_handle_link_event(struct i40e_pf *pf,
9467 struct i40e_arq_event_info *e)
9469 struct i40e_aqc_get_link_status *status =
9470 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9472 /* Do a new status request to re-enable LSE reporting
9473 * and load new status information into the hw struct
9474 * This completely ignores any state information
9475 * in the ARQ event info, instead choosing to always
9476 * issue the AQ update link status command.
9478 i40e_link_event(pf);
9480 /* Check if module meets thermal requirements */
9481 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9482 dev_err(&pf->pdev->dev,
9483 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9484 dev_err(&pf->pdev->dev,
9485 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9487 /* check for unqualified module, if link is down, suppress
9488 * the message if link was forced to be down.
9490 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9491 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9492 (!(status->link_info & I40E_AQ_LINK_UP)) &&
9493 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9494 dev_err(&pf->pdev->dev,
9495 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9496 dev_err(&pf->pdev->dev,
9497 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9503 * i40e_clean_adminq_subtask - Clean the AdminQ rings
9504 * @pf: board private structure
9506 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9508 struct i40e_arq_event_info event;
9509 struct i40e_hw *hw = &pf->hw;
9516 /* Do not run clean AQ when PF reset fails */
9517 if (test_bit(__I40E_RESET_FAILED, pf->state))
9520 /* check for error indications */
9521 val = rd32(&pf->hw, pf->hw.aq.arq.len);
9523 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9524 if (hw->debug_mask & I40E_DEBUG_AQ)
9525 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9526 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9528 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9529 if (hw->debug_mask & I40E_DEBUG_AQ)
9530 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9531 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9532 pf->arq_overflows++;
9534 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9535 if (hw->debug_mask & I40E_DEBUG_AQ)
9536 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9537 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9540 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9542 val = rd32(&pf->hw, pf->hw.aq.asq.len);
9544 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9545 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9546 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9547 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9549 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9550 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9551 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9552 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9554 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9555 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9556 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9557 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9560 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9562 event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9563 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9568 ret = i40e_clean_arq_element(hw, &event, &pending);
9569 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9572 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9576 opcode = le16_to_cpu(event.desc.opcode);
9579 case i40e_aqc_opc_get_link_status:
9580 i40e_handle_link_event(pf, &event);
9582 case i40e_aqc_opc_send_msg_to_pf:
9583 ret = i40e_vc_process_vf_msg(pf,
9584 le16_to_cpu(event.desc.retval),
9585 le32_to_cpu(event.desc.cookie_high),
9586 le32_to_cpu(event.desc.cookie_low),
9590 case i40e_aqc_opc_lldp_update_mib:
9591 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9592 #ifdef CONFIG_I40E_DCB
9594 ret = i40e_handle_lldp_event(pf, &event);
9596 #endif /* CONFIG_I40E_DCB */
9598 case i40e_aqc_opc_event_lan_overflow:
9599 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9600 i40e_handle_lan_overflow_event(pf, &event);
9602 case i40e_aqc_opc_send_msg_to_peer:
9603 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9605 case i40e_aqc_opc_nvm_erase:
9606 case i40e_aqc_opc_nvm_update:
9607 case i40e_aqc_opc_oem_post_update:
9608 i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9609 "ARQ NVM operation 0x%04x completed\n",
9613 dev_info(&pf->pdev->dev,
9614 "ARQ: Unknown event 0x%04x ignored\n",
9618 } while (i++ < pf->adminq_work_limit);
9620 if (i < pf->adminq_work_limit)
9621 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9623 /* re-enable Admin queue interrupt cause */
9624 val = rd32(hw, I40E_PFINT_ICR0_ENA);
9625 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9626 wr32(hw, I40E_PFINT_ICR0_ENA, val);
9629 kfree(event.msg_buf);
9633 * i40e_verify_eeprom - make sure eeprom is good to use
9634 * @pf: board private structure
9636 static void i40e_verify_eeprom(struct i40e_pf *pf)
9640 err = i40e_diag_eeprom_test(&pf->hw);
9642 /* retry in case of garbage read */
9643 err = i40e_diag_eeprom_test(&pf->hw);
9645 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9647 set_bit(__I40E_BAD_EEPROM, pf->state);
9651 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9652 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9653 clear_bit(__I40E_BAD_EEPROM, pf->state);
9658 * i40e_enable_pf_switch_lb
9659 * @pf: pointer to the PF structure
9661 * enable switch loop back or die - no point in a return value
9663 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
9665 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9666 struct i40e_vsi_context ctxt;
9669 ctxt.seid = pf->main_vsi_seid;
9670 ctxt.pf_num = pf->hw.pf_id;
9672 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9674 dev_info(&pf->pdev->dev,
9675 "couldn't get PF vsi config, err %s aq_err %s\n",
9676 i40e_stat_str(&pf->hw, ret),
9677 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9680 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9681 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9682 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9684 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9686 dev_info(&pf->pdev->dev,
9687 "update vsi switch failed, err %s aq_err %s\n",
9688 i40e_stat_str(&pf->hw, ret),
9689 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9694 * i40e_disable_pf_switch_lb
9695 * @pf: pointer to the PF structure
9697 * disable switch loop back or die - no point in a return value
9699 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
9701 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9702 struct i40e_vsi_context ctxt;
9705 ctxt.seid = pf->main_vsi_seid;
9706 ctxt.pf_num = pf->hw.pf_id;
9708 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9710 dev_info(&pf->pdev->dev,
9711 "couldn't get PF vsi config, err %s aq_err %s\n",
9712 i40e_stat_str(&pf->hw, ret),
9713 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9716 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9717 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9718 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9720 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9722 dev_info(&pf->pdev->dev,
9723 "update vsi switch failed, err %s aq_err %s\n",
9724 i40e_stat_str(&pf->hw, ret),
9725 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9730 * i40e_config_bridge_mode - Configure the HW bridge mode
9731 * @veb: pointer to the bridge instance
9733 * Configure the loop back mode for the LAN VSI that is downlink to the
9734 * specified HW bridge instance. It is expected this function is called
9735 * when a new HW bridge is instantiated.
9737 static void i40e_config_bridge_mode(struct i40e_veb *veb)
9739 struct i40e_pf *pf = veb->pf;
9741 if (pf->hw.debug_mask & I40E_DEBUG_LAN)
9742 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
9743 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9744 if (veb->bridge_mode & BRIDGE_MODE_VEPA)
9745 i40e_disable_pf_switch_lb(pf);
9747 i40e_enable_pf_switch_lb(pf);
9751 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
9752 * @veb: pointer to the VEB instance
9754 * This is a recursive function that first builds the attached VSIs then
9755 * recurses in to build the next layer of VEB. We track the connections
9756 * through our own index numbers because the seid's from the HW could
9757 * change across the reset.
9759 static int i40e_reconstitute_veb(struct i40e_veb *veb)
9761 struct i40e_vsi *ctl_vsi = NULL;
9762 struct i40e_pf *pf = veb->pf;
9766 /* build VSI that owns this VEB, temporarily attached to base VEB */
9767 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
9769 pf->vsi[v]->veb_idx == veb->idx &&
9770 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
9771 ctl_vsi = pf->vsi[v];
9776 dev_info(&pf->pdev->dev,
9777 "missing owner VSI for veb_idx %d\n", veb->idx);
9779 goto end_reconstitute;
9781 if (ctl_vsi != pf->vsi[pf->lan_vsi])
9782 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
9783 ret = i40e_add_vsi(ctl_vsi);
9785 dev_info(&pf->pdev->dev,
9786 "rebuild of veb_idx %d owner VSI failed: %d\n",
9788 goto end_reconstitute;
9790 i40e_vsi_reset_stats(ctl_vsi);
9792 /* create the VEB in the switch and move the VSI onto the VEB */
9793 ret = i40e_add_veb(veb, ctl_vsi);
9795 goto end_reconstitute;
9797 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
9798 veb->bridge_mode = BRIDGE_MODE_VEB;
9800 veb->bridge_mode = BRIDGE_MODE_VEPA;
9801 i40e_config_bridge_mode(veb);
9803 /* create the remaining VSIs attached to this VEB */
9804 for (v = 0; v < pf->num_alloc_vsi; v++) {
9805 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
9808 if (pf->vsi[v]->veb_idx == veb->idx) {
9809 struct i40e_vsi *vsi = pf->vsi[v];
9811 vsi->uplink_seid = veb->seid;
9812 ret = i40e_add_vsi(vsi);
9814 dev_info(&pf->pdev->dev,
9815 "rebuild of vsi_idx %d failed: %d\n",
9817 goto end_reconstitute;
9819 i40e_vsi_reset_stats(vsi);
9823 /* create any VEBs attached to this VEB - RECURSION */
9824 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
9825 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
9826 pf->veb[veb_idx]->uplink_seid = veb->seid;
9827 ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
9838 * i40e_get_capabilities - get info about the HW
9839 * @pf: the PF struct
9840 * @list_type: AQ capability to be queried
9842 static int i40e_get_capabilities(struct i40e_pf *pf,
9843 enum i40e_admin_queue_opc list_type)
9845 struct i40e_aqc_list_capabilities_element_resp *cap_buf;
9850 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
9852 cap_buf = kzalloc(buf_len, GFP_KERNEL);
9856 /* this loads the data into the hw struct for us */
9857 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
9858 &data_size, list_type,
9860 /* data loaded, buffer no longer needed */
9863 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
9864 /* retry with a larger buffer */
9865 buf_len = data_size;
9866 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
9867 dev_info(&pf->pdev->dev,
9868 "capability discovery failed, err %s aq_err %s\n",
9869 i40e_stat_str(&pf->hw, err),
9870 i40e_aq_str(&pf->hw,
9871 pf->hw.aq.asq_last_status));
9876 if (pf->hw.debug_mask & I40E_DEBUG_USER) {
9877 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9878 dev_info(&pf->pdev->dev,
9879 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
9880 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
9881 pf->hw.func_caps.num_msix_vectors,
9882 pf->hw.func_caps.num_msix_vectors_vf,
9883 pf->hw.func_caps.fd_filters_guaranteed,
9884 pf->hw.func_caps.fd_filters_best_effort,
9885 pf->hw.func_caps.num_tx_qp,
9886 pf->hw.func_caps.num_vsis);
9887 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
9888 dev_info(&pf->pdev->dev,
9889 "switch_mode=0x%04x, function_valid=0x%08x\n",
9890 pf->hw.dev_caps.switch_mode,
9891 pf->hw.dev_caps.valid_functions);
9892 dev_info(&pf->pdev->dev,
9893 "SR-IOV=%d, num_vfs for all function=%u\n",
9894 pf->hw.dev_caps.sr_iov_1_1,
9895 pf->hw.dev_caps.num_vfs);
9896 dev_info(&pf->pdev->dev,
9897 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
9898 pf->hw.dev_caps.num_vsis,
9899 pf->hw.dev_caps.num_rx_qp,
9900 pf->hw.dev_caps.num_tx_qp);
9903 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9904 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
9905 + pf->hw.func_caps.num_vfs)
9906 if (pf->hw.revision_id == 0 &&
9907 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
9908 dev_info(&pf->pdev->dev,
9909 "got num_vsis %d, setting num_vsis to %d\n",
9910 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
9911 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
9917 static int i40e_vsi_clear(struct i40e_vsi *vsi);
9920 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
9921 * @pf: board private structure
9923 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
9925 struct i40e_vsi *vsi;
9927 /* quick workaround for an NVM issue that leaves a critical register
9930 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
9931 static const u32 hkey[] = {
9932 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
9933 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
9934 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
9938 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
9939 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
9942 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9945 /* find existing VSI and see if it needs configuring */
9946 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9948 /* create a new VSI if none exists */
9950 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
9951 pf->vsi[pf->lan_vsi]->seid, 0);
9953 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
9954 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
9955 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
9960 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
9964 * i40e_fdir_teardown - release the Flow Director resources
9965 * @pf: board private structure
9967 static void i40e_fdir_teardown(struct i40e_pf *pf)
9969 struct i40e_vsi *vsi;
9971 i40e_fdir_filter_exit(pf);
9972 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9974 i40e_vsi_release(vsi);
9978 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
9980 * @seid: seid of main or channel VSIs
9982 * Rebuilds cloud filters associated with main VSI and channel VSIs if they
9983 * existed before reset
9985 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
9987 struct i40e_cloud_filter *cfilter;
9988 struct i40e_pf *pf = vsi->back;
9989 struct hlist_node *node;
9992 /* Add cloud filters back if they exist */
9993 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
9995 if (cfilter->seid != seid)
9998 if (cfilter->dst_port)
9999 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10002 ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10005 dev_dbg(&pf->pdev->dev,
10006 "Failed to rebuild cloud filter, err %s aq_err %s\n",
10007 i40e_stat_str(&pf->hw, ret),
10008 i40e_aq_str(&pf->hw,
10009 pf->hw.aq.asq_last_status));
10017 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10018 * @vsi: PF main vsi
10020 * Rebuilds channel VSIs if they existed before reset
10022 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10024 struct i40e_channel *ch, *ch_tmp;
10027 if (list_empty(&vsi->ch_list))
10030 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10031 if (!ch->initialized)
10033 /* Proceed with creation of channel (VMDq2) VSI */
10034 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10036 dev_info(&vsi->back->pdev->dev,
10037 "failed to rebuild channels using uplink_seid %u\n",
10041 /* Reconfigure TX queues using QTX_CTL register */
10042 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10044 dev_info(&vsi->back->pdev->dev,
10045 "failed to configure TX rings for channel %u\n",
10049 /* update 'next_base_queue' */
10050 vsi->next_base_queue = vsi->next_base_queue +
10051 ch->num_queue_pairs;
10052 if (ch->max_tx_rate) {
10053 u64 credits = ch->max_tx_rate;
10055 if (i40e_set_bw_limit(vsi, ch->seid,
10059 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10060 dev_dbg(&vsi->back->pdev->dev,
10061 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10066 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10068 dev_dbg(&vsi->back->pdev->dev,
10069 "Failed to rebuild cloud filters for channel VSI %u\n",
10078 * i40e_clean_xps_state - clean xps state for every tx_ring
10079 * @vsi: ptr to the VSI
10081 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10086 for (i = 0; i < vsi->num_queue_pairs; i++)
10087 if (vsi->tx_rings[i])
10088 clear_bit(__I40E_TX_XPS_INIT_DONE,
10089 vsi->tx_rings[i]->state);
10093 * i40e_prep_for_reset - prep for the core to reset
10094 * @pf: board private structure
10095 * @lock_acquired: indicates whether or not the lock has been acquired
10096 * before this function was called.
10098 * Close up the VFs and other things in prep for PF Reset.
10100 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
10102 struct i40e_hw *hw = &pf->hw;
10103 i40e_status ret = 0;
10106 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10107 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10109 if (i40e_check_asq_alive(&pf->hw))
10110 i40e_vc_notify_reset(pf);
10112 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10114 /* quiesce the VSIs and their queues that are not already DOWN */
10115 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
10116 if (!lock_acquired)
10118 i40e_pf_quiesce_all_vsi(pf);
10119 if (!lock_acquired)
10122 for (v = 0; v < pf->num_alloc_vsi; v++) {
10124 i40e_clean_xps_state(pf->vsi[v]);
10125 pf->vsi[v]->seid = 0;
10129 i40e_shutdown_adminq(&pf->hw);
10131 /* call shutdown HMC */
10132 if (hw->hmc.hmc_obj) {
10133 ret = i40e_shutdown_lan_hmc(hw);
10135 dev_warn(&pf->pdev->dev,
10136 "shutdown_lan_hmc failed: %d\n", ret);
10139 /* Save the current PTP time so that we can restore the time after the
10142 i40e_ptp_save_hw_time(pf);
10146 * i40e_send_version - update firmware with driver version
10149 static void i40e_send_version(struct i40e_pf *pf)
10151 struct i40e_driver_version dv;
10153 dv.major_version = 0xff;
10154 dv.minor_version = 0xff;
10155 dv.build_version = 0xff;
10156 dv.subbuild_version = 0;
10157 strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10158 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10162 * i40e_get_oem_version - get OEM specific version information
10163 * @hw: pointer to the hardware structure
10165 static void i40e_get_oem_version(struct i40e_hw *hw)
10167 u16 block_offset = 0xffff;
10168 u16 block_length = 0;
10169 u16 capabilities = 0;
10173 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B
10174 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00
10175 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01
10176 #define I40E_NVM_OEM_GEN_OFFSET 0x02
10177 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03
10178 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F
10179 #define I40E_NVM_OEM_LENGTH 3
10181 /* Check if pointer to OEM version block is valid. */
10182 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10183 if (block_offset == 0xffff)
10186 /* Check if OEM version block has correct length. */
10187 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10189 if (block_length < I40E_NVM_OEM_LENGTH)
10192 /* Check if OEM version format is as expected. */
10193 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10195 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10198 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10200 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10202 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10203 hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10207 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10208 * @pf: board private structure
10210 static int i40e_reset(struct i40e_pf *pf)
10212 struct i40e_hw *hw = &pf->hw;
10215 ret = i40e_pf_reset(hw);
10217 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10218 set_bit(__I40E_RESET_FAILED, pf->state);
10219 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10227 * i40e_rebuild - rebuild using a saved config
10228 * @pf: board private structure
10229 * @reinit: if the Main VSI needs to re-initialized.
10230 * @lock_acquired: indicates whether or not the lock has been acquired
10231 * before this function was called.
10233 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10235 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10236 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10237 struct i40e_hw *hw = &pf->hw;
10242 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10243 is_recovery_mode_reported)
10244 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10246 if (test_bit(__I40E_DOWN, pf->state) &&
10247 !test_bit(__I40E_RECOVERY_MODE, pf->state))
10248 goto clear_recovery;
10249 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10251 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10252 ret = i40e_init_adminq(&pf->hw);
10254 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
10255 i40e_stat_str(&pf->hw, ret),
10256 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10257 goto clear_recovery;
10259 i40e_get_oem_version(&pf->hw);
10261 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10262 /* The following delay is necessary for firmware update. */
10266 /* re-verify the eeprom if we just had an EMP reset */
10267 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10268 i40e_verify_eeprom(pf);
10270 /* if we are going out of or into recovery mode we have to act
10271 * accordingly with regard to resources initialization
10272 * and deinitialization
10274 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10275 if (i40e_get_capabilities(pf,
10276 i40e_aqc_opc_list_func_capabilities))
10279 if (is_recovery_mode_reported) {
10280 /* we're staying in recovery mode so we'll reinitialize
10283 if (i40e_setup_misc_vector_for_recovery_mode(pf))
10286 if (!lock_acquired)
10288 /* we're going out of recovery mode so we'll free
10289 * the IRQ allocated specifically for recovery mode
10290 * and restore the interrupt scheme
10292 free_irq(pf->pdev->irq, pf);
10293 i40e_clear_interrupt_scheme(pf);
10294 if (i40e_restore_interrupt_scheme(pf))
10298 /* tell the firmware that we're starting */
10299 i40e_send_version(pf);
10301 /* bail out in case recovery mode was detected, as there is
10302 * no need for further configuration.
10307 i40e_clear_pxe_mode(hw);
10308 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10310 goto end_core_reset;
10312 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10313 hw->func_caps.num_rx_qp, 0, 0);
10315 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10316 goto end_core_reset;
10318 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10320 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10321 goto end_core_reset;
10324 /* Enable FW to write a default DCB config on link-up */
10325 i40e_aq_set_dcb_parameters(hw, true, NULL);
10327 #ifdef CONFIG_I40E_DCB
10328 ret = i40e_init_pf_dcb(pf);
10330 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
10331 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10332 /* Continue without DCB enabled */
10334 #endif /* CONFIG_I40E_DCB */
10335 /* do basic switch setup */
10336 if (!lock_acquired)
10338 ret = i40e_setup_pf_switch(pf, reinit, true);
10342 /* The driver only wants link up/down and module qualification
10343 * reports from firmware. Note the negative logic.
10345 ret = i40e_aq_set_phy_int_mask(&pf->hw,
10346 ~(I40E_AQ_EVENT_LINK_UPDOWN |
10347 I40E_AQ_EVENT_MEDIA_NA |
10348 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10350 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10351 i40e_stat_str(&pf->hw, ret),
10352 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10354 /* Rebuild the VSIs and VEBs that existed before reset.
10355 * They are still in our local switch element arrays, so only
10356 * need to rebuild the switch model in the HW.
10358 * If there were VEBs but the reconstitution failed, we'll try
10359 * try to recover minimal use by getting the basic PF VSI working.
10361 if (vsi->uplink_seid != pf->mac_seid) {
10362 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10363 /* find the one VEB connected to the MAC, and find orphans */
10364 for (v = 0; v < I40E_MAX_VEB; v++) {
10368 if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10369 pf->veb[v]->uplink_seid == 0) {
10370 ret = i40e_reconstitute_veb(pf->veb[v]);
10375 /* If Main VEB failed, we're in deep doodoo,
10376 * so give up rebuilding the switch and set up
10377 * for minimal rebuild of PF VSI.
10378 * If orphan failed, we'll report the error
10379 * but try to keep going.
10381 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10382 dev_info(&pf->pdev->dev,
10383 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10385 vsi->uplink_seid = pf->mac_seid;
10387 } else if (pf->veb[v]->uplink_seid == 0) {
10388 dev_info(&pf->pdev->dev,
10389 "rebuild of orphan VEB failed: %d\n",
10396 if (vsi->uplink_seid == pf->mac_seid) {
10397 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10398 /* no VEB, so rebuild only the Main VSI */
10399 ret = i40e_add_vsi(vsi);
10401 dev_info(&pf->pdev->dev,
10402 "rebuild of Main VSI failed: %d\n", ret);
10407 if (vsi->mqprio_qopt.max_rate[0]) {
10408 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
10409 vsi->mqprio_qopt.max_rate[0]);
10412 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10416 credits = max_tx_rate;
10417 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10418 dev_dbg(&vsi->back->pdev->dev,
10419 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10425 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10429 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
10430 * for this main VSI if they exist
10432 ret = i40e_rebuild_channels(vsi);
10436 /* Reconfigure hardware for allowing smaller MSS in the case
10437 * of TSO, so that we avoid the MDD being fired and causing
10438 * a reset in the case of small MSS+TSO.
10440 #define I40E_REG_MSS 0x000E64DC
10441 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
10442 #define I40E_64BYTE_MSS 0x400000
10443 val = rd32(hw, I40E_REG_MSS);
10444 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10445 val &= ~I40E_REG_MSS_MIN_MASK;
10446 val |= I40E_64BYTE_MSS;
10447 wr32(hw, I40E_REG_MSS, val);
10450 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10452 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10454 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10455 i40e_stat_str(&pf->hw, ret),
10456 i40e_aq_str(&pf->hw,
10457 pf->hw.aq.asq_last_status));
10459 /* reinit the misc interrupt */
10460 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
10461 ret = i40e_setup_misc_vector(pf);
10466 /* Add a filter to drop all Flow control frames from any VSI from being
10467 * transmitted. By doing so we stop a malicious VF from sending out
10468 * PAUSE or PFC frames and potentially controlling traffic for other
10470 * The FW can still send Flow control frames if enabled.
10472 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10473 pf->main_vsi_seid);
10475 /* restart the VSIs that were rebuilt and running before the reset */
10476 i40e_pf_unquiesce_all_vsi(pf);
10478 /* Release the RTNL lock before we start resetting VFs */
10479 if (!lock_acquired)
10482 /* Restore promiscuous settings */
10483 ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10485 dev_warn(&pf->pdev->dev,
10486 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10487 pf->cur_promisc ? "on" : "off",
10488 i40e_stat_str(&pf->hw, ret),
10489 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10491 i40e_reset_all_vfs(pf, true);
10493 /* tell the firmware that we're starting */
10494 i40e_send_version(pf);
10496 /* We've already released the lock, so don't do it again */
10497 goto end_core_reset;
10500 if (!lock_acquired)
10503 clear_bit(__I40E_RESET_FAILED, pf->state);
10505 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10506 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10510 * i40e_reset_and_rebuild - reset and rebuild using a saved config
10511 * @pf: board private structure
10512 * @reinit: if the Main VSI needs to re-initialized.
10513 * @lock_acquired: indicates whether or not the lock has been acquired
10514 * before this function was called.
10516 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10517 bool lock_acquired)
10520 /* Now we wait for GRST to settle out.
10521 * We don't have to delete the VEBs or VSIs from the hw switch
10522 * because the reset will make them disappear.
10524 ret = i40e_reset(pf);
10526 i40e_rebuild(pf, reinit, lock_acquired);
10530 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
10531 * @pf: board private structure
10533 * Close up the VFs and other things in prep for a Core Reset,
10534 * then get ready to rebuild the world.
10535 * @lock_acquired: indicates whether or not the lock has been acquired
10536 * before this function was called.
10538 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10540 i40e_prep_for_reset(pf, lock_acquired);
10541 i40e_reset_and_rebuild(pf, false, lock_acquired);
10545 * i40e_handle_mdd_event
10546 * @pf: pointer to the PF structure
10548 * Called from the MDD irq handler to identify possibly malicious vfs
10550 static void i40e_handle_mdd_event(struct i40e_pf *pf)
10552 struct i40e_hw *hw = &pf->hw;
10553 bool mdd_detected = false;
10554 struct i40e_vf *vf;
10558 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10561 /* find what triggered the MDD event */
10562 reg = rd32(hw, I40E_GL_MDET_TX);
10563 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10564 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10565 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10566 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10567 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10568 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10569 I40E_GL_MDET_TX_EVENT_SHIFT;
10570 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10571 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10572 pf->hw.func_caps.base_queue;
10573 if (netif_msg_tx_err(pf))
10574 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10575 event, queue, pf_num, vf_num);
10576 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10577 mdd_detected = true;
10579 reg = rd32(hw, I40E_GL_MDET_RX);
10580 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10581 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10582 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10583 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10584 I40E_GL_MDET_RX_EVENT_SHIFT;
10585 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10586 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10587 pf->hw.func_caps.base_queue;
10588 if (netif_msg_rx_err(pf))
10589 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10590 event, queue, func);
10591 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10592 mdd_detected = true;
10595 if (mdd_detected) {
10596 reg = rd32(hw, I40E_PF_MDET_TX);
10597 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10598 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10599 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10601 reg = rd32(hw, I40E_PF_MDET_RX);
10602 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10603 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10604 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10608 /* see if one of the VFs needs its hand slapped */
10609 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10611 reg = rd32(hw, I40E_VP_MDET_TX(i));
10612 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10613 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10614 vf->num_mdd_events++;
10615 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10617 dev_info(&pf->pdev->dev,
10618 "Use PF Control I/F to re-enable the VF\n");
10619 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10622 reg = rd32(hw, I40E_VP_MDET_RX(i));
10623 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10624 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10625 vf->num_mdd_events++;
10626 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10628 dev_info(&pf->pdev->dev,
10629 "Use PF Control I/F to re-enable the VF\n");
10630 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10634 /* re-enable mdd interrupt cause */
10635 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10636 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10637 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10638 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10643 * i40e_service_task - Run the driver's async subtasks
10644 * @work: pointer to work_struct containing our data
10646 static void i40e_service_task(struct work_struct *work)
10648 struct i40e_pf *pf = container_of(work,
10651 unsigned long start_time = jiffies;
10653 /* don't bother with service tasks if a reset is in progress */
10654 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10655 test_bit(__I40E_SUSPENDED, pf->state))
10658 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10661 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10662 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10663 i40e_sync_filters_subtask(pf);
10664 i40e_reset_subtask(pf);
10665 i40e_handle_mdd_event(pf);
10666 i40e_vc_process_vflr_event(pf);
10667 i40e_watchdog_subtask(pf);
10668 i40e_fdir_reinit_subtask(pf);
10669 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10670 /* Client subtask will reopen next time through. */
10671 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
10674 i40e_client_subtask(pf);
10675 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10677 i40e_notify_client_of_l2_param_changes(
10678 pf->vsi[pf->lan_vsi]);
10680 i40e_sync_filters_subtask(pf);
10682 i40e_reset_subtask(pf);
10685 i40e_clean_adminq_subtask(pf);
10687 /* flush memory to make sure state is correct before next watchdog */
10688 smp_mb__before_atomic();
10689 clear_bit(__I40E_SERVICE_SCHED, pf->state);
10691 /* If the tasks have taken longer than one timer cycle or there
10692 * is more work to be done, reschedule the service task now
10693 * rather than wait for the timer to tick again.
10695 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10696 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
10697 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
10698 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10699 i40e_service_event_schedule(pf);
10703 * i40e_service_timer - timer callback
10704 * @t: timer list pointer
10706 static void i40e_service_timer(struct timer_list *t)
10708 struct i40e_pf *pf = from_timer(pf, t, service_timer);
10710 mod_timer(&pf->service_timer,
10711 round_jiffies(jiffies + pf->service_timer_period));
10712 i40e_service_event_schedule(pf);
10716 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
10717 * @vsi: the VSI being configured
10719 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
10721 struct i40e_pf *pf = vsi->back;
10723 switch (vsi->type) {
10724 case I40E_VSI_MAIN:
10725 vsi->alloc_queue_pairs = pf->num_lan_qps;
10726 if (!vsi->num_tx_desc)
10727 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10728 I40E_REQ_DESCRIPTOR_MULTIPLE);
10729 if (!vsi->num_rx_desc)
10730 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10731 I40E_REQ_DESCRIPTOR_MULTIPLE);
10732 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10733 vsi->num_q_vectors = pf->num_lan_msix;
10735 vsi->num_q_vectors = 1;
10739 case I40E_VSI_FDIR:
10740 vsi->alloc_queue_pairs = 1;
10741 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10742 I40E_REQ_DESCRIPTOR_MULTIPLE);
10743 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10744 I40E_REQ_DESCRIPTOR_MULTIPLE);
10745 vsi->num_q_vectors = pf->num_fdsb_msix;
10748 case I40E_VSI_VMDQ2:
10749 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
10750 if (!vsi->num_tx_desc)
10751 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10752 I40E_REQ_DESCRIPTOR_MULTIPLE);
10753 if (!vsi->num_rx_desc)
10754 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10755 I40E_REQ_DESCRIPTOR_MULTIPLE);
10756 vsi->num_q_vectors = pf->num_vmdq_msix;
10759 case I40E_VSI_SRIOV:
10760 vsi->alloc_queue_pairs = pf->num_vf_qps;
10761 if (!vsi->num_tx_desc)
10762 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10763 I40E_REQ_DESCRIPTOR_MULTIPLE);
10764 if (!vsi->num_rx_desc)
10765 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10766 I40E_REQ_DESCRIPTOR_MULTIPLE);
10778 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
10779 * @vsi: VSI pointer
10780 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
10782 * On error: returns error code (negative)
10783 * On success: returns 0
10785 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
10787 struct i40e_ring **next_rings;
10791 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
10792 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
10793 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
10794 vsi->tx_rings = kzalloc(size, GFP_KERNEL);
10795 if (!vsi->tx_rings)
10797 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
10798 if (i40e_enabled_xdp_vsi(vsi)) {
10799 vsi->xdp_rings = next_rings;
10800 next_rings += vsi->alloc_queue_pairs;
10802 vsi->rx_rings = next_rings;
10804 if (alloc_qvectors) {
10805 /* allocate memory for q_vector pointers */
10806 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
10807 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
10808 if (!vsi->q_vectors) {
10816 kfree(vsi->tx_rings);
10821 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
10822 * @pf: board private structure
10823 * @type: type of VSI
10825 * On error: returns error code (negative)
10826 * On success: returns vsi index in PF (positive)
10828 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
10831 struct i40e_vsi *vsi;
10835 /* Need to protect the allocation of the VSIs at the PF level */
10836 mutex_lock(&pf->switch_mutex);
10838 /* VSI list may be fragmented if VSI creation/destruction has
10839 * been happening. We can afford to do a quick scan to look
10840 * for any free VSIs in the list.
10842 * find next empty vsi slot, looping back around if necessary
10845 while (i < pf->num_alloc_vsi && pf->vsi[i])
10847 if (i >= pf->num_alloc_vsi) {
10849 while (i < pf->next_vsi && pf->vsi[i])
10853 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
10854 vsi_idx = i; /* Found one! */
10857 goto unlock_pf; /* out of VSI slots! */
10859 pf->next_vsi = ++i;
10861 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
10868 set_bit(__I40E_VSI_DOWN, vsi->state);
10870 vsi->idx = vsi_idx;
10871 vsi->int_rate_limit = 0;
10872 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
10873 pf->rss_table_size : 64;
10874 vsi->netdev_registered = false;
10875 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
10876 hash_init(vsi->mac_filter_hash);
10877 vsi->irqs_ready = false;
10879 if (type == I40E_VSI_MAIN) {
10880 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
10881 if (!vsi->af_xdp_zc_qps)
10885 ret = i40e_set_num_rings_in_vsi(vsi);
10889 ret = i40e_vsi_alloc_arrays(vsi, true);
10893 /* Setup default MSIX irq handler for VSI */
10894 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
10896 /* Initialize VSI lock */
10897 spin_lock_init(&vsi->mac_filter_hash_lock);
10898 pf->vsi[vsi_idx] = vsi;
10903 bitmap_free(vsi->af_xdp_zc_qps);
10904 pf->next_vsi = i - 1;
10907 mutex_unlock(&pf->switch_mutex);
10912 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
10913 * @vsi: VSI pointer
10914 * @free_qvectors: a bool to specify if q_vectors need to be freed.
10916 * On error: returns error code (negative)
10917 * On success: returns 0
10919 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
10921 /* free the ring and vector containers */
10922 if (free_qvectors) {
10923 kfree(vsi->q_vectors);
10924 vsi->q_vectors = NULL;
10926 kfree(vsi->tx_rings);
10927 vsi->tx_rings = NULL;
10928 vsi->rx_rings = NULL;
10929 vsi->xdp_rings = NULL;
10933 * i40e_clear_rss_config_user - clear the user configured RSS hash keys
10935 * @vsi: Pointer to VSI structure
10937 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
10942 kfree(vsi->rss_hkey_user);
10943 vsi->rss_hkey_user = NULL;
10945 kfree(vsi->rss_lut_user);
10946 vsi->rss_lut_user = NULL;
10950 * i40e_vsi_clear - Deallocate the VSI provided
10951 * @vsi: the VSI being un-configured
10953 static int i40e_vsi_clear(struct i40e_vsi *vsi)
10955 struct i40e_pf *pf;
10964 mutex_lock(&pf->switch_mutex);
10965 if (!pf->vsi[vsi->idx]) {
10966 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
10967 vsi->idx, vsi->idx, vsi->type);
10971 if (pf->vsi[vsi->idx] != vsi) {
10972 dev_err(&pf->pdev->dev,
10973 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
10974 pf->vsi[vsi->idx]->idx,
10975 pf->vsi[vsi->idx]->type,
10976 vsi->idx, vsi->type);
10980 /* updates the PF for this cleared vsi */
10981 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10982 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
10984 bitmap_free(vsi->af_xdp_zc_qps);
10985 i40e_vsi_free_arrays(vsi, true);
10986 i40e_clear_rss_config_user(vsi);
10988 pf->vsi[vsi->idx] = NULL;
10989 if (vsi->idx < pf->next_vsi)
10990 pf->next_vsi = vsi->idx;
10993 mutex_unlock(&pf->switch_mutex);
11001 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11002 * @vsi: the VSI being cleaned
11004 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11008 if (vsi->tx_rings && vsi->tx_rings[0]) {
11009 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11010 kfree_rcu(vsi->tx_rings[i], rcu);
11011 WRITE_ONCE(vsi->tx_rings[i], NULL);
11012 WRITE_ONCE(vsi->rx_rings[i], NULL);
11013 if (vsi->xdp_rings)
11014 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11020 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11021 * @vsi: the VSI being configured
11023 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11025 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11026 struct i40e_pf *pf = vsi->back;
11027 struct i40e_ring *ring;
11029 /* Set basic values in the rings to be used later during open() */
11030 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11031 /* allocate space for both Tx and Rx in one shot */
11032 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11036 ring->queue_index = i;
11037 ring->reg_idx = vsi->base_queue + i;
11038 ring->ring_active = false;
11040 ring->netdev = vsi->netdev;
11041 ring->dev = &pf->pdev->dev;
11042 ring->count = vsi->num_tx_desc;
11045 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11046 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11047 ring->itr_setting = pf->tx_itr_default;
11048 WRITE_ONCE(vsi->tx_rings[i], ring++);
11050 if (!i40e_enabled_xdp_vsi(vsi))
11053 ring->queue_index = vsi->alloc_queue_pairs + i;
11054 ring->reg_idx = vsi->base_queue + ring->queue_index;
11055 ring->ring_active = false;
11057 ring->netdev = NULL;
11058 ring->dev = &pf->pdev->dev;
11059 ring->count = vsi->num_tx_desc;
11062 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11063 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11064 set_ring_xdp(ring);
11065 ring->itr_setting = pf->tx_itr_default;
11066 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11069 ring->queue_index = i;
11070 ring->reg_idx = vsi->base_queue + i;
11071 ring->ring_active = false;
11073 ring->netdev = vsi->netdev;
11074 ring->dev = &pf->pdev->dev;
11075 ring->count = vsi->num_rx_desc;
11078 ring->itr_setting = pf->rx_itr_default;
11079 WRITE_ONCE(vsi->rx_rings[i], ring);
11085 i40e_vsi_clear_rings(vsi);
11090 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11091 * @pf: board private structure
11092 * @vectors: the number of MSI-X vectors to request
11094 * Returns the number of vectors reserved, or error
11096 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11098 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11099 I40E_MIN_MSIX, vectors);
11101 dev_info(&pf->pdev->dev,
11102 "MSI-X vector reservation failed: %d\n", vectors);
11110 * i40e_init_msix - Setup the MSIX capability
11111 * @pf: board private structure
11113 * Work with the OS to set up the MSIX vectors needed.
11115 * Returns the number of vectors reserved or negative on failure
11117 static int i40e_init_msix(struct i40e_pf *pf)
11119 struct i40e_hw *hw = &pf->hw;
11120 int cpus, extra_vectors;
11124 int iwarp_requested = 0;
11126 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11129 /* The number of vectors we'll request will be comprised of:
11130 * - Add 1 for "other" cause for Admin Queue events, etc.
11131 * - The number of LAN queue pairs
11132 * - Queues being used for RSS.
11133 * We don't need as many as max_rss_size vectors.
11134 * use rss_size instead in the calculation since that
11135 * is governed by number of cpus in the system.
11136 * - assumes symmetric Tx/Rx pairing
11137 * - The number of VMDq pairs
11138 * - The CPU count within the NUMA node if iWARP is enabled
11139 * Once we count this up, try the request.
11141 * If we can't get what we want, we'll simplify to nearly nothing
11142 * and try again. If that still fails, we punt.
11144 vectors_left = hw->func_caps.num_msix_vectors;
11147 /* reserve one vector for miscellaneous handler */
11148 if (vectors_left) {
11153 /* reserve some vectors for the main PF traffic queues. Initially we
11154 * only reserve at most 50% of the available vectors, in the case that
11155 * the number of online CPUs is large. This ensures that we can enable
11156 * extra features as well. Once we've enabled the other features, we
11157 * will use any remaining vectors to reach as close as we can to the
11158 * number of online CPUs.
11160 cpus = num_online_cpus();
11161 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11162 vectors_left -= pf->num_lan_msix;
11164 /* reserve one vector for sideband flow director */
11165 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11166 if (vectors_left) {
11167 pf->num_fdsb_msix = 1;
11171 pf->num_fdsb_msix = 0;
11175 /* can we reserve enough for iWARP? */
11176 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11177 iwarp_requested = pf->num_iwarp_msix;
11180 pf->num_iwarp_msix = 0;
11181 else if (vectors_left < pf->num_iwarp_msix)
11182 pf->num_iwarp_msix = 1;
11183 v_budget += pf->num_iwarp_msix;
11184 vectors_left -= pf->num_iwarp_msix;
11187 /* any vectors left over go for VMDq support */
11188 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11189 if (!vectors_left) {
11190 pf->num_vmdq_msix = 0;
11191 pf->num_vmdq_qps = 0;
11193 int vmdq_vecs_wanted =
11194 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11196 min_t(int, vectors_left, vmdq_vecs_wanted);
11198 /* if we're short on vectors for what's desired, we limit
11199 * the queues per vmdq. If this is still more than are
11200 * available, the user will need to change the number of
11201 * queues/vectors used by the PF later with the ethtool
11204 if (vectors_left < vmdq_vecs_wanted) {
11205 pf->num_vmdq_qps = 1;
11206 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11207 vmdq_vecs = min_t(int,
11211 pf->num_vmdq_msix = pf->num_vmdq_qps;
11213 v_budget += vmdq_vecs;
11214 vectors_left -= vmdq_vecs;
11218 /* On systems with a large number of SMP cores, we previously limited
11219 * the number of vectors for num_lan_msix to be at most 50% of the
11220 * available vectors, to allow for other features. Now, we add back
11221 * the remaining vectors. However, we ensure that the total
11222 * num_lan_msix will not exceed num_online_cpus(). To do this, we
11223 * calculate the number of vectors we can add without going over the
11224 * cap of CPUs. For systems with a small number of CPUs this will be
11227 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11228 pf->num_lan_msix += extra_vectors;
11229 vectors_left -= extra_vectors;
11231 WARN(vectors_left < 0,
11232 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11234 v_budget += pf->num_lan_msix;
11235 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11237 if (!pf->msix_entries)
11240 for (i = 0; i < v_budget; i++)
11241 pf->msix_entries[i].entry = i;
11242 v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11244 if (v_actual < I40E_MIN_MSIX) {
11245 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11246 kfree(pf->msix_entries);
11247 pf->msix_entries = NULL;
11248 pci_disable_msix(pf->pdev);
11251 } else if (v_actual == I40E_MIN_MSIX) {
11252 /* Adjust for minimal MSIX use */
11253 pf->num_vmdq_vsis = 0;
11254 pf->num_vmdq_qps = 0;
11255 pf->num_lan_qps = 1;
11256 pf->num_lan_msix = 1;
11258 } else if (v_actual != v_budget) {
11259 /* If we have limited resources, we will start with no vectors
11260 * for the special features and then allocate vectors to some
11261 * of these features based on the policy and at the end disable
11262 * the features that did not get any vectors.
11266 dev_info(&pf->pdev->dev,
11267 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11268 v_actual, v_budget);
11269 /* reserve the misc vector */
11270 vec = v_actual - 1;
11272 /* Scale vector usage down */
11273 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
11274 pf->num_vmdq_vsis = 1;
11275 pf->num_vmdq_qps = 1;
11277 /* partition out the remaining vectors */
11280 pf->num_lan_msix = 1;
11283 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11284 pf->num_lan_msix = 1;
11285 pf->num_iwarp_msix = 1;
11287 pf->num_lan_msix = 2;
11291 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11292 pf->num_iwarp_msix = min_t(int, (vec / 3),
11294 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11295 I40E_DEFAULT_NUM_VMDQ_VSI);
11297 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11298 I40E_DEFAULT_NUM_VMDQ_VSI);
11300 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11301 pf->num_fdsb_msix = 1;
11304 pf->num_lan_msix = min_t(int,
11305 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11307 pf->num_lan_qps = pf->num_lan_msix;
11312 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11313 (pf->num_fdsb_msix == 0)) {
11314 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11315 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11316 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11318 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11319 (pf->num_vmdq_msix == 0)) {
11320 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11321 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11324 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11325 (pf->num_iwarp_msix == 0)) {
11326 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11327 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11329 i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11330 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11332 pf->num_vmdq_msix * pf->num_vmdq_vsis,
11334 pf->num_iwarp_msix);
11340 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11341 * @vsi: the VSI being configured
11342 * @v_idx: index of the vector in the vsi struct
11344 * We allocate one q_vector. If allocation fails we return -ENOMEM.
11346 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11348 struct i40e_q_vector *q_vector;
11350 /* allocate q_vector */
11351 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11355 q_vector->vsi = vsi;
11356 q_vector->v_idx = v_idx;
11357 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11360 netif_napi_add(vsi->netdev, &q_vector->napi,
11361 i40e_napi_poll, NAPI_POLL_WEIGHT);
11363 /* tie q_vector and vsi together */
11364 vsi->q_vectors[v_idx] = q_vector;
11370 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11371 * @vsi: the VSI being configured
11373 * We allocate one q_vector per queue interrupt. If allocation fails we
11376 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11378 struct i40e_pf *pf = vsi->back;
11379 int err, v_idx, num_q_vectors;
11381 /* if not MSIX, give the one vector only to the LAN VSI */
11382 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11383 num_q_vectors = vsi->num_q_vectors;
11384 else if (vsi == pf->vsi[pf->lan_vsi])
11389 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11390 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
11399 i40e_free_q_vector(vsi, v_idx);
11405 * i40e_init_interrupt_scheme - Determine proper interrupt scheme
11406 * @pf: board private structure to initialize
11408 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11413 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11414 vectors = i40e_init_msix(pf);
11416 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED |
11417 I40E_FLAG_IWARP_ENABLED |
11418 I40E_FLAG_RSS_ENABLED |
11419 I40E_FLAG_DCB_CAPABLE |
11420 I40E_FLAG_DCB_ENABLED |
11421 I40E_FLAG_SRIOV_ENABLED |
11422 I40E_FLAG_FD_SB_ENABLED |
11423 I40E_FLAG_FD_ATR_ENABLED |
11424 I40E_FLAG_VMDQ_ENABLED);
11425 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11427 /* rework the queue expectations without MSIX */
11428 i40e_determine_queue_usage(pf);
11432 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11433 (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11434 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11435 vectors = pci_enable_msi(pf->pdev);
11437 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11439 pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11441 vectors = 1; /* one MSI or Legacy vector */
11444 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11445 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11447 /* set up vector assignment tracking */
11448 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11449 pf->irq_pile = kzalloc(size, GFP_KERNEL);
11453 pf->irq_pile->num_entries = vectors;
11455 /* track first vector for misc interrupts, ignore return */
11456 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11462 * i40e_restore_interrupt_scheme - Restore the interrupt scheme
11463 * @pf: private board data structure
11465 * Restore the interrupt scheme that was cleared when we suspended the
11466 * device. This should be called during resume to re-allocate the q_vectors
11467 * and reacquire IRQs.
11469 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11473 /* We cleared the MSI and MSI-X flags when disabling the old interrupt
11474 * scheme. We need to re-enabled them here in order to attempt to
11475 * re-acquire the MSI or MSI-X vectors
11477 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11479 err = i40e_init_interrupt_scheme(pf);
11483 /* Now that we've re-acquired IRQs, we need to remap the vectors and
11484 * rings together again.
11486 for (i = 0; i < pf->num_alloc_vsi; i++) {
11488 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11491 i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11495 err = i40e_setup_misc_vector(pf);
11499 if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11500 i40e_client_update_msix_info(pf);
11507 i40e_vsi_free_q_vectors(pf->vsi[i]);
11514 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
11515 * non queue events in recovery mode
11516 * @pf: board private structure
11518 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
11519 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
11520 * This is handled differently than in recovery mode since no Tx/Rx resources
11521 * are being allocated.
11523 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11527 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11528 err = i40e_setup_misc_vector(pf);
11531 dev_info(&pf->pdev->dev,
11532 "MSI-X misc vector request failed, error %d\n",
11537 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11539 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11543 dev_info(&pf->pdev->dev,
11544 "MSI/legacy misc vector request failed, error %d\n",
11548 i40e_enable_misc_int_causes(pf);
11549 i40e_irq_dynamic_enable_icr0(pf);
11556 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
11557 * @pf: board private structure
11559 * This sets up the handler for MSIX 0, which is used to manage the
11560 * non-queue interrupts, e.g. AdminQ and errors. This is not used
11561 * when in MSI or Legacy interrupt mode.
11563 static int i40e_setup_misc_vector(struct i40e_pf *pf)
11565 struct i40e_hw *hw = &pf->hw;
11568 /* Only request the IRQ once, the first time through. */
11569 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11570 err = request_irq(pf->msix_entries[0].vector,
11571 i40e_intr, 0, pf->int_name, pf);
11573 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11574 dev_info(&pf->pdev->dev,
11575 "request_irq for %s failed: %d\n",
11576 pf->int_name, err);
11581 i40e_enable_misc_int_causes(pf);
11583 /* associate no queues to the misc vector */
11584 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11585 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
11589 i40e_irq_dynamic_enable_icr0(pf);
11595 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
11596 * @vsi: Pointer to vsi structure
11597 * @seed: Buffter to store the hash keys
11598 * @lut: Buffer to store the lookup table entries
11599 * @lut_size: Size of buffer to store the lookup table entries
11601 * Return 0 on success, negative on failure
11603 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11604 u8 *lut, u16 lut_size)
11606 struct i40e_pf *pf = vsi->back;
11607 struct i40e_hw *hw = &pf->hw;
11611 ret = i40e_aq_get_rss_key(hw, vsi->id,
11612 (struct i40e_aqc_get_set_rss_key_data *)seed);
11614 dev_info(&pf->pdev->dev,
11615 "Cannot get RSS key, err %s aq_err %s\n",
11616 i40e_stat_str(&pf->hw, ret),
11617 i40e_aq_str(&pf->hw,
11618 pf->hw.aq.asq_last_status));
11624 bool pf_lut = vsi->type == I40E_VSI_MAIN;
11626 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11628 dev_info(&pf->pdev->dev,
11629 "Cannot get RSS lut, err %s aq_err %s\n",
11630 i40e_stat_str(&pf->hw, ret),
11631 i40e_aq_str(&pf->hw,
11632 pf->hw.aq.asq_last_status));
11641 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
11642 * @vsi: Pointer to vsi structure
11643 * @seed: RSS hash seed
11644 * @lut: Lookup table
11645 * @lut_size: Lookup table size
11647 * Returns 0 on success, negative on failure
11649 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11650 const u8 *lut, u16 lut_size)
11652 struct i40e_pf *pf = vsi->back;
11653 struct i40e_hw *hw = &pf->hw;
11654 u16 vf_id = vsi->vf_id;
11657 /* Fill out hash function seed */
11659 u32 *seed_dw = (u32 *)seed;
11661 if (vsi->type == I40E_VSI_MAIN) {
11662 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11663 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
11664 } else if (vsi->type == I40E_VSI_SRIOV) {
11665 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
11666 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
11668 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
11673 u32 *lut_dw = (u32 *)lut;
11675 if (vsi->type == I40E_VSI_MAIN) {
11676 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11678 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11679 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
11680 } else if (vsi->type == I40E_VSI_SRIOV) {
11681 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
11683 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11684 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
11686 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11695 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
11696 * @vsi: Pointer to VSI structure
11697 * @seed: Buffer to store the keys
11698 * @lut: Buffer to store the lookup table entries
11699 * @lut_size: Size of buffer to store the lookup table entries
11701 * Returns 0 on success, negative on failure
11703 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
11704 u8 *lut, u16 lut_size)
11706 struct i40e_pf *pf = vsi->back;
11707 struct i40e_hw *hw = &pf->hw;
11711 u32 *seed_dw = (u32 *)seed;
11713 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11714 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
11717 u32 *lut_dw = (u32 *)lut;
11719 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11721 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11722 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
11729 * i40e_config_rss - Configure RSS keys and lut
11730 * @vsi: Pointer to VSI structure
11731 * @seed: RSS hash seed
11732 * @lut: Lookup table
11733 * @lut_size: Lookup table size
11735 * Returns 0 on success, negative on failure
11737 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11739 struct i40e_pf *pf = vsi->back;
11741 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11742 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
11744 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
11748 * i40e_get_rss - Get RSS keys and lut
11749 * @vsi: Pointer to VSI structure
11750 * @seed: Buffer to store the keys
11751 * @lut: Buffer to store the lookup table entries
11752 * @lut_size: Size of buffer to store the lookup table entries
11754 * Returns 0 on success, negative on failure
11756 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11758 struct i40e_pf *pf = vsi->back;
11760 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11761 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
11763 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
11767 * i40e_fill_rss_lut - Fill the RSS lookup table with default values
11768 * @pf: Pointer to board private structure
11769 * @lut: Lookup table
11770 * @rss_table_size: Lookup table size
11771 * @rss_size: Range of queue number for hashing
11773 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
11774 u16 rss_table_size, u16 rss_size)
11778 for (i = 0; i < rss_table_size; i++)
11779 lut[i] = i % rss_size;
11783 * i40e_pf_config_rss - Prepare for RSS if used
11784 * @pf: board private structure
11786 static int i40e_pf_config_rss(struct i40e_pf *pf)
11788 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11789 u8 seed[I40E_HKEY_ARRAY_SIZE];
11791 struct i40e_hw *hw = &pf->hw;
11796 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
11797 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
11798 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
11799 hena |= i40e_pf_get_default_rss_hena(pf);
11801 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
11802 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
11804 /* Determine the RSS table size based on the hardware capabilities */
11805 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
11806 reg_val = (pf->rss_table_size == 512) ?
11807 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
11808 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
11809 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
11811 /* Determine the RSS size of the VSI */
11812 if (!vsi->rss_size) {
11814 /* If the firmware does something weird during VSI init, we
11815 * could end up with zero TCs. Check for that to avoid
11816 * divide-by-zero. It probably won't pass traffic, but it also
11819 qcount = vsi->num_queue_pairs /
11820 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
11821 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11823 if (!vsi->rss_size)
11826 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
11830 /* Use user configured lut if there is one, otherwise use default */
11831 if (vsi->rss_lut_user)
11832 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
11834 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
11836 /* Use user configured hash key if there is one, otherwise
11839 if (vsi->rss_hkey_user)
11840 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
11842 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
11843 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
11850 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
11851 * @pf: board private structure
11852 * @queue_count: the requested queue count for rss.
11854 * returns 0 if rss is not enabled, if enabled returns the final rss queue
11855 * count which may be different from the requested queue count.
11856 * Note: expects to be called while under rtnl_lock()
11858 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
11860 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11863 if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
11866 queue_count = min_t(int, queue_count, num_online_cpus());
11867 new_rss_size = min_t(int, queue_count, pf->rss_size_max);
11869 if (queue_count != vsi->num_queue_pairs) {
11872 vsi->req_queue_pairs = queue_count;
11873 i40e_prep_for_reset(pf, true);
11875 pf->alloc_rss_size = new_rss_size;
11877 i40e_reset_and_rebuild(pf, true, true);
11879 /* Discard the user configured hash keys and lut, if less
11880 * queues are enabled.
11882 if (queue_count < vsi->rss_size) {
11883 i40e_clear_rss_config_user(vsi);
11884 dev_dbg(&pf->pdev->dev,
11885 "discard user configured hash keys and lut\n");
11888 /* Reset vsi->rss_size, as number of enabled queues changed */
11889 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
11890 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11892 i40e_pf_config_rss(pf);
11894 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
11895 vsi->req_queue_pairs, pf->rss_size_max);
11896 return pf->alloc_rss_size;
11900 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
11901 * @pf: board private structure
11903 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
11905 i40e_status status;
11906 bool min_valid, max_valid;
11907 u32 max_bw, min_bw;
11909 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
11910 &min_valid, &max_valid);
11914 pf->min_bw = min_bw;
11916 pf->max_bw = max_bw;
11923 * i40e_set_partition_bw_setting - Set BW settings for this PF partition
11924 * @pf: board private structure
11926 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
11928 struct i40e_aqc_configure_partition_bw_data bw_data;
11929 i40e_status status;
11931 memset(&bw_data, 0, sizeof(bw_data));
11933 /* Set the valid bit for this PF */
11934 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
11935 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
11936 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
11938 /* Set the new bandwidths */
11939 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
11945 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
11946 * @pf: board private structure
11948 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
11950 /* Commit temporary BW setting to permanent NVM image */
11951 enum i40e_admin_queue_err last_aq_status;
11955 if (pf->hw.partition_id != 1) {
11956 dev_info(&pf->pdev->dev,
11957 "Commit BW only works on partition 1! This is partition %d",
11958 pf->hw.partition_id);
11959 ret = I40E_NOT_SUPPORTED;
11960 goto bw_commit_out;
11963 /* Acquire NVM for read access */
11964 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
11965 last_aq_status = pf->hw.aq.asq_last_status;
11967 dev_info(&pf->pdev->dev,
11968 "Cannot acquire NVM for read access, err %s aq_err %s\n",
11969 i40e_stat_str(&pf->hw, ret),
11970 i40e_aq_str(&pf->hw, last_aq_status));
11971 goto bw_commit_out;
11974 /* Read word 0x10 of NVM - SW compatibility word 1 */
11975 ret = i40e_aq_read_nvm(&pf->hw,
11976 I40E_SR_NVM_CONTROL_WORD,
11977 0x10, sizeof(nvm_word), &nvm_word,
11979 /* Save off last admin queue command status before releasing
11982 last_aq_status = pf->hw.aq.asq_last_status;
11983 i40e_release_nvm(&pf->hw);
11985 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
11986 i40e_stat_str(&pf->hw, ret),
11987 i40e_aq_str(&pf->hw, last_aq_status));
11988 goto bw_commit_out;
11991 /* Wait a bit for NVM release to complete */
11994 /* Acquire NVM for write access */
11995 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
11996 last_aq_status = pf->hw.aq.asq_last_status;
11998 dev_info(&pf->pdev->dev,
11999 "Cannot acquire NVM for write access, err %s aq_err %s\n",
12000 i40e_stat_str(&pf->hw, ret),
12001 i40e_aq_str(&pf->hw, last_aq_status));
12002 goto bw_commit_out;
12004 /* Write it back out unchanged to initiate update NVM,
12005 * which will force a write of the shadow (alt) RAM to
12006 * the NVM - thus storing the bandwidth values permanently.
12008 ret = i40e_aq_update_nvm(&pf->hw,
12009 I40E_SR_NVM_CONTROL_WORD,
12010 0x10, sizeof(nvm_word),
12011 &nvm_word, true, 0, NULL);
12012 /* Save off last admin queue command status before releasing
12015 last_aq_status = pf->hw.aq.asq_last_status;
12016 i40e_release_nvm(&pf->hw);
12018 dev_info(&pf->pdev->dev,
12019 "BW settings NOT SAVED, err %s aq_err %s\n",
12020 i40e_stat_str(&pf->hw, ret),
12021 i40e_aq_str(&pf->hw, last_aq_status));
12028 * i40e_is_total_port_shutdown_enabled - read NVM and return value
12029 * if total port shutdown feature is enabled for this PF
12030 * @pf: board private structure
12032 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12034 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4)
12035 #define I40E_FEATURES_ENABLE_PTR 0x2A
12036 #define I40E_CURRENT_SETTING_PTR 0x2B
12037 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D
12038 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1
12039 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0)
12040 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4
12041 i40e_status read_status = I40E_SUCCESS;
12042 u16 sr_emp_sr_settings_ptr = 0;
12043 u16 features_enable = 0;
12044 u16 link_behavior = 0;
12047 read_status = i40e_read_nvm_word(&pf->hw,
12048 I40E_SR_EMP_SR_SETTINGS_PTR,
12049 &sr_emp_sr_settings_ptr);
12052 read_status = i40e_read_nvm_word(&pf->hw,
12053 sr_emp_sr_settings_ptr +
12054 I40E_FEATURES_ENABLE_PTR,
12058 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12059 read_status = i40e_read_nvm_module_data(&pf->hw,
12060 I40E_SR_EMP_SR_SETTINGS_PTR,
12061 I40E_CURRENT_SETTING_PTR,
12062 I40E_LINK_BEHAVIOR_WORD_OFFSET,
12063 I40E_LINK_BEHAVIOR_WORD_LENGTH,
12067 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12068 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12073 dev_warn(&pf->pdev->dev,
12074 "total-port-shutdown feature is off due to read nvm error: %s\n",
12075 i40e_stat_str(&pf->hw, read_status));
12080 * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12081 * @pf: board private structure to initialize
12083 * i40e_sw_init initializes the Adapter private data structure.
12084 * Fields are initialized based on PCI device information and
12085 * OS network device settings (MTU size).
12087 static int i40e_sw_init(struct i40e_pf *pf)
12093 /* Set default capability flags */
12094 pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12095 I40E_FLAG_MSI_ENABLED |
12096 I40E_FLAG_MSIX_ENABLED;
12098 /* Set default ITR */
12099 pf->rx_itr_default = I40E_ITR_RX_DEF;
12100 pf->tx_itr_default = I40E_ITR_TX_DEF;
12102 /* Depending on PF configurations, it is possible that the RSS
12103 * maximum might end up larger than the available queues
12105 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12106 pf->alloc_rss_size = 1;
12107 pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12108 pf->rss_size_max = min_t(int, pf->rss_size_max,
12109 pf->hw.func_caps.num_tx_qp);
12111 /* find the next higher power-of-2 of num cpus */
12112 pow = roundup_pow_of_two(num_online_cpus());
12113 pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12115 if (pf->hw.func_caps.rss) {
12116 pf->flags |= I40E_FLAG_RSS_ENABLED;
12117 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12118 num_online_cpus());
12121 /* MFP mode enabled */
12122 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12123 pf->flags |= I40E_FLAG_MFP_ENABLED;
12124 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12125 if (i40e_get_partition_bw_setting(pf)) {
12126 dev_warn(&pf->pdev->dev,
12127 "Could not get partition bw settings\n");
12129 dev_info(&pf->pdev->dev,
12130 "Partition BW Min = %8.8x, Max = %8.8x\n",
12131 pf->min_bw, pf->max_bw);
12133 /* nudge the Tx scheduler */
12134 i40e_set_partition_bw_setting(pf);
12138 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12139 (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12140 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12141 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12142 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12143 pf->hw.num_partitions > 1)
12144 dev_info(&pf->pdev->dev,
12145 "Flow Director Sideband mode Disabled in MFP mode\n");
12147 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12148 pf->fdir_pf_filter_count =
12149 pf->hw.func_caps.fd_filters_guaranteed;
12150 pf->hw.fdir_shared_filter_count =
12151 pf->hw.func_caps.fd_filters_best_effort;
12154 if (pf->hw.mac.type == I40E_MAC_X722) {
12155 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12156 I40E_HW_128_QP_RSS_CAPABLE |
12157 I40E_HW_ATR_EVICT_CAPABLE |
12158 I40E_HW_WB_ON_ITR_CAPABLE |
12159 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12160 I40E_HW_NO_PCI_LINK_CHECK |
12161 I40E_HW_USE_SET_LLDP_MIB |
12162 I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12163 I40E_HW_PTP_L4_CAPABLE |
12164 I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12165 I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12167 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12168 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12169 I40E_FDEVICT_PCTYPE_DEFAULT) {
12170 dev_warn(&pf->pdev->dev,
12171 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12172 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12174 } else if ((pf->hw.aq.api_maj_ver > 1) ||
12175 ((pf->hw.aq.api_maj_ver == 1) &&
12176 (pf->hw.aq.api_min_ver > 4))) {
12177 /* Supported in FW API version higher than 1.4 */
12178 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12181 /* Enable HW ATR eviction if possible */
12182 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12183 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12185 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12186 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12187 (pf->hw.aq.fw_maj_ver < 4))) {
12188 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12189 /* No DCB support for FW < v4.33 */
12190 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12193 /* Disable FW LLDP if FW < v4.3 */
12194 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12195 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12196 (pf->hw.aq.fw_maj_ver < 4)))
12197 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12199 /* Use the FW Set LLDP MIB API if FW > v4.40 */
12200 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12201 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12202 (pf->hw.aq.fw_maj_ver >= 5)))
12203 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12205 /* Enable PTP L4 if FW > v6.0 */
12206 if (pf->hw.mac.type == I40E_MAC_XL710 &&
12207 pf->hw.aq.fw_maj_ver >= 6)
12208 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12210 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12211 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12212 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12213 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12216 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12217 pf->flags |= I40E_FLAG_IWARP_ENABLED;
12218 /* IWARP needs one extra vector for CQP just like MISC.*/
12219 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12221 /* Stopping FW LLDP engine is supported on XL710 and X722
12222 * starting from FW versions determined in i40e_init_adminq.
12223 * Stopping the FW LLDP engine is not supported on XL710
12224 * if NPAR is functioning so unset this hw flag in this case.
12226 if (pf->hw.mac.type == I40E_MAC_XL710 &&
12227 pf->hw.func_caps.npar_enable &&
12228 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12229 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12231 #ifdef CONFIG_PCI_IOV
12232 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12233 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12234 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12235 pf->num_req_vfs = min_t(int,
12236 pf->hw.func_caps.num_vfs,
12237 I40E_MAX_VF_COUNT);
12239 #endif /* CONFIG_PCI_IOV */
12240 pf->eeprom_version = 0xDEAD;
12241 pf->lan_veb = I40E_NO_VEB;
12242 pf->lan_vsi = I40E_NO_VSI;
12244 /* By default FW has this off for performance reasons */
12245 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12247 /* set up queue assignment tracking */
12248 size = sizeof(struct i40e_lump_tracking)
12249 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12250 pf->qp_pile = kzalloc(size, GFP_KERNEL);
12251 if (!pf->qp_pile) {
12255 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12257 pf->tx_timeout_recovery_level = 1;
12259 if (pf->hw.mac.type != I40E_MAC_X722 &&
12260 i40e_is_total_port_shutdown_enabled(pf)) {
12261 /* Link down on close must be on when total port shutdown
12262 * is enabled for a given port
12264 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12265 I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12266 dev_info(&pf->pdev->dev,
12267 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12269 mutex_init(&pf->switch_mutex);
12276 * i40e_set_ntuple - set the ntuple feature flag and take action
12277 * @pf: board private structure to initialize
12278 * @features: the feature set that the stack is suggesting
12280 * returns a bool to indicate if reset needs to happen
12282 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12284 bool need_reset = false;
12286 /* Check if Flow Director n-tuple support was enabled or disabled. If
12287 * the state changed, we need to reset.
12289 if (features & NETIF_F_NTUPLE) {
12290 /* Enable filters and mark for reset */
12291 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12293 /* enable FD_SB only if there is MSI-X vector and no cloud
12296 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12297 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12298 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12301 /* turn off filters, mark for reset and clear SW filter list */
12302 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12304 i40e_fdir_filter_exit(pf);
12306 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12307 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12308 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12310 /* reset fd counters */
12311 pf->fd_add_err = 0;
12312 pf->fd_atr_cnt = 0;
12313 /* if ATR was auto disabled it can be re-enabled. */
12314 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12315 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12316 (I40E_DEBUG_FD & pf->hw.debug_mask))
12317 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12323 * i40e_clear_rss_lut - clear the rx hash lookup table
12324 * @vsi: the VSI being configured
12326 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12328 struct i40e_pf *pf = vsi->back;
12329 struct i40e_hw *hw = &pf->hw;
12330 u16 vf_id = vsi->vf_id;
12333 if (vsi->type == I40E_VSI_MAIN) {
12334 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12335 wr32(hw, I40E_PFQF_HLUT(i), 0);
12336 } else if (vsi->type == I40E_VSI_SRIOV) {
12337 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12338 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12340 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12345 * i40e_set_features - set the netdev feature flags
12346 * @netdev: ptr to the netdev being adjusted
12347 * @features: the feature set that the stack is suggesting
12348 * Note: expects to be called while under rtnl_lock()
12350 static int i40e_set_features(struct net_device *netdev,
12351 netdev_features_t features)
12353 struct i40e_netdev_priv *np = netdev_priv(netdev);
12354 struct i40e_vsi *vsi = np->vsi;
12355 struct i40e_pf *pf = vsi->back;
12358 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12359 i40e_pf_config_rss(pf);
12360 else if (!(features & NETIF_F_RXHASH) &&
12361 netdev->features & NETIF_F_RXHASH)
12362 i40e_clear_rss_lut(vsi);
12364 if (features & NETIF_F_HW_VLAN_CTAG_RX)
12365 i40e_vlan_stripping_enable(vsi);
12367 i40e_vlan_stripping_disable(vsi);
12369 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12370 dev_err(&pf->pdev->dev,
12371 "Offloaded tc filters active, can't turn hw_tc_offload off");
12375 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12376 i40e_del_all_macvlans(vsi);
12378 need_reset = i40e_set_ntuple(pf, features);
12381 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12386 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12387 unsigned int table, unsigned int idx,
12388 struct udp_tunnel_info *ti)
12390 struct i40e_netdev_priv *np = netdev_priv(netdev);
12391 struct i40e_hw *hw = &np->vsi->back->hw;
12392 u8 type, filter_index;
12395 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12396 I40E_AQC_TUNNEL_TYPE_NGE;
12398 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
12401 netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n",
12402 i40e_stat_str(hw, ret),
12403 i40e_aq_str(hw, hw->aq.asq_last_status));
12407 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
12411 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12412 unsigned int table, unsigned int idx,
12413 struct udp_tunnel_info *ti)
12415 struct i40e_netdev_priv *np = netdev_priv(netdev);
12416 struct i40e_hw *hw = &np->vsi->back->hw;
12419 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
12421 netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n",
12422 i40e_stat_str(hw, ret),
12423 i40e_aq_str(hw, hw->aq.asq_last_status));
12430 static int i40e_get_phys_port_id(struct net_device *netdev,
12431 struct netdev_phys_item_id *ppid)
12433 struct i40e_netdev_priv *np = netdev_priv(netdev);
12434 struct i40e_pf *pf = np->vsi->back;
12435 struct i40e_hw *hw = &pf->hw;
12437 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12438 return -EOPNOTSUPP;
12440 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12441 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12447 * i40e_ndo_fdb_add - add an entry to the hardware database
12448 * @ndm: the input from the stack
12449 * @tb: pointer to array of nladdr (unused)
12450 * @dev: the net device pointer
12451 * @addr: the MAC address entry being added
12453 * @flags: instructions from stack about fdb operation
12454 * @extack: netlink extended ack, unused currently
12456 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12457 struct net_device *dev,
12458 const unsigned char *addr, u16 vid,
12460 struct netlink_ext_ack *extack)
12462 struct i40e_netdev_priv *np = netdev_priv(dev);
12463 struct i40e_pf *pf = np->vsi->back;
12466 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12467 return -EOPNOTSUPP;
12470 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12474 /* Hardware does not support aging addresses so if a
12475 * ndm_state is given only allow permanent addresses
12477 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12478 netdev_info(dev, "FDB only supports static addresses\n");
12482 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12483 err = dev_uc_add_excl(dev, addr);
12484 else if (is_multicast_ether_addr(addr))
12485 err = dev_mc_add_excl(dev, addr);
12489 /* Only return duplicate errors if NLM_F_EXCL is set */
12490 if (err == -EEXIST && !(flags & NLM_F_EXCL))
12497 * i40e_ndo_bridge_setlink - Set the hardware bridge mode
12498 * @dev: the netdev being configured
12499 * @nlh: RTNL message
12500 * @flags: bridge flags
12501 * @extack: netlink extended ack
12503 * Inserts a new hardware bridge if not already created and
12504 * enables the bridging mode requested (VEB or VEPA). If the
12505 * hardware bridge has already been inserted and the request
12506 * is to change the mode then that requires a PF reset to
12507 * allow rebuild of the components with required hardware
12508 * bridge mode enabled.
12510 * Note: expects to be called while under rtnl_lock()
12512 static int i40e_ndo_bridge_setlink(struct net_device *dev,
12513 struct nlmsghdr *nlh,
12515 struct netlink_ext_ack *extack)
12517 struct i40e_netdev_priv *np = netdev_priv(dev);
12518 struct i40e_vsi *vsi = np->vsi;
12519 struct i40e_pf *pf = vsi->back;
12520 struct i40e_veb *veb = NULL;
12521 struct nlattr *attr, *br_spec;
12524 /* Only for PF VSI for now */
12525 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12526 return -EOPNOTSUPP;
12528 /* Find the HW bridge for PF VSI */
12529 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12530 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12534 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12538 nla_for_each_nested(attr, br_spec, rem) {
12541 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12544 mode = nla_get_u16(attr);
12545 if ((mode != BRIDGE_MODE_VEPA) &&
12546 (mode != BRIDGE_MODE_VEB))
12549 /* Insert a new HW bridge */
12551 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12552 vsi->tc_config.enabled_tc);
12554 veb->bridge_mode = mode;
12555 i40e_config_bridge_mode(veb);
12557 /* No Bridge HW offload available */
12561 } else if (mode != veb->bridge_mode) {
12562 /* Existing HW bridge but different mode needs reset */
12563 veb->bridge_mode = mode;
12564 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
12565 if (mode == BRIDGE_MODE_VEB)
12566 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12568 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12569 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12578 * i40e_ndo_bridge_getlink - Get the hardware bridge mode
12581 * @seq: RTNL message seq #
12582 * @dev: the netdev being configured
12583 * @filter_mask: unused
12584 * @nlflags: netlink flags passed in
12586 * Return the mode in which the hardware bridge is operating in
12589 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12590 struct net_device *dev,
12591 u32 __always_unused filter_mask,
12594 struct i40e_netdev_priv *np = netdev_priv(dev);
12595 struct i40e_vsi *vsi = np->vsi;
12596 struct i40e_pf *pf = vsi->back;
12597 struct i40e_veb *veb = NULL;
12600 /* Only for PF VSI for now */
12601 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12602 return -EOPNOTSUPP;
12604 /* Find the HW bridge for the PF VSI */
12605 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12606 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12613 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12614 0, 0, nlflags, filter_mask, NULL);
12618 * i40e_features_check - Validate encapsulated packet conforms to limits
12620 * @dev: This physical port's netdev
12621 * @features: Offload features that the stack believes apply
12623 static netdev_features_t i40e_features_check(struct sk_buff *skb,
12624 struct net_device *dev,
12625 netdev_features_t features)
12629 /* No point in doing any of this if neither checksum nor GSO are
12630 * being requested for this frame. We can rule out both by just
12631 * checking for CHECKSUM_PARTIAL
12633 if (skb->ip_summed != CHECKSUM_PARTIAL)
12636 /* We cannot support GSO if the MSS is going to be less than
12637 * 64 bytes. If it is then we need to drop support for GSO.
12639 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12640 features &= ~NETIF_F_GSO_MASK;
12642 /* MACLEN can support at most 63 words */
12643 len = skb_network_header(skb) - skb->data;
12644 if (len & ~(63 * 2))
12647 /* IPLEN and EIPLEN can support at most 127 dwords */
12648 len = skb_transport_header(skb) - skb_network_header(skb);
12649 if (len & ~(127 * 4))
12652 if (skb->encapsulation) {
12653 /* L4TUNLEN can support 127 words */
12654 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12655 if (len & ~(127 * 2))
12658 /* IPLEN can support at most 127 dwords */
12659 len = skb_inner_transport_header(skb) -
12660 skb_inner_network_header(skb);
12661 if (len & ~(127 * 4))
12665 /* No need to validate L4LEN as TCP is the only protocol with a
12666 * a flexible value and we support all possible values supported
12667 * by TCP, which is at most 15 dwords
12672 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12676 * i40e_xdp_setup - add/remove an XDP program
12677 * @vsi: VSI to changed
12678 * @prog: XDP program
12680 static int i40e_xdp_setup(struct i40e_vsi *vsi,
12681 struct bpf_prog *prog)
12683 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
12684 struct i40e_pf *pf = vsi->back;
12685 struct bpf_prog *old_prog;
12689 /* Don't allow frames that span over multiple buffers */
12690 if (frame_size > vsi->rx_buf_len)
12693 if (!i40e_enabled_xdp_vsi(vsi) && !prog)
12696 /* When turning XDP on->off/off->on we reset and rebuild the rings. */
12697 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
12700 i40e_prep_for_reset(pf, true);
12702 old_prog = xchg(&vsi->xdp_prog, prog);
12706 /* Wait until ndo_xsk_wakeup completes. */
12708 i40e_reset_and_rebuild(pf, true, true);
12711 if (!i40e_enabled_xdp_vsi(vsi) && prog) {
12712 if (i40e_realloc_rx_bi_zc(vsi, true))
12714 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
12715 if (i40e_realloc_rx_bi_zc(vsi, false))
12719 for (i = 0; i < vsi->num_queue_pairs; i++)
12720 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
12723 bpf_prog_put(old_prog);
12725 /* Kick start the NAPI context if there is an AF_XDP socket open
12726 * on that queue id. This so that receiving will start.
12728 if (need_reset && prog)
12729 for (i = 0; i < vsi->num_queue_pairs; i++)
12730 if (vsi->xdp_rings[i]->xsk_pool)
12731 (void)i40e_xsk_wakeup(vsi->netdev, i,
12738 * i40e_enter_busy_conf - Enters busy config state
12741 * Returns 0 on success, <0 for failure.
12743 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
12745 struct i40e_pf *pf = vsi->back;
12748 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
12752 usleep_range(1000, 2000);
12759 * i40e_exit_busy_conf - Exits busy config state
12762 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
12764 struct i40e_pf *pf = vsi->back;
12766 clear_bit(__I40E_CONFIG_BUSY, pf->state);
12770 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
12772 * @queue_pair: queue pair
12774 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
12776 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
12777 sizeof(vsi->rx_rings[queue_pair]->rx_stats));
12778 memset(&vsi->tx_rings[queue_pair]->stats, 0,
12779 sizeof(vsi->tx_rings[queue_pair]->stats));
12780 if (i40e_enabled_xdp_vsi(vsi)) {
12781 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
12782 sizeof(vsi->xdp_rings[queue_pair]->stats));
12787 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
12789 * @queue_pair: queue pair
12791 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
12793 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
12794 if (i40e_enabled_xdp_vsi(vsi)) {
12795 /* Make sure that in-progress ndo_xdp_xmit calls are
12799 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
12801 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
12805 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
12807 * @queue_pair: queue pair
12808 * @enable: true for enable, false for disable
12810 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
12813 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12814 struct i40e_q_vector *q_vector = rxr->q_vector;
12819 /* All rings in a qp belong to the same qvector. */
12820 if (q_vector->rx.ring || q_vector->tx.ring) {
12822 napi_enable(&q_vector->napi);
12824 napi_disable(&q_vector->napi);
12829 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
12831 * @queue_pair: queue pair
12832 * @enable: true for enable, false for disable
12834 * Returns 0 on success, <0 on failure.
12836 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
12839 struct i40e_pf *pf = vsi->back;
12842 pf_q = vsi->base_queue + queue_pair;
12843 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
12844 false /*is xdp*/, enable);
12846 dev_info(&pf->pdev->dev,
12847 "VSI seid %d Tx ring %d %sable timeout\n",
12848 vsi->seid, pf_q, (enable ? "en" : "dis"));
12852 i40e_control_rx_q(pf, pf_q, enable);
12853 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
12855 dev_info(&pf->pdev->dev,
12856 "VSI seid %d Rx ring %d %sable timeout\n",
12857 vsi->seid, pf_q, (enable ? "en" : "dis"));
12861 /* Due to HW errata, on Rx disable only, the register can
12862 * indicate done before it really is. Needs 50ms to be sure
12867 if (!i40e_enabled_xdp_vsi(vsi))
12870 ret = i40e_control_wait_tx_q(vsi->seid, pf,
12871 pf_q + vsi->alloc_queue_pairs,
12872 true /*is xdp*/, enable);
12874 dev_info(&pf->pdev->dev,
12875 "VSI seid %d XDP Tx ring %d %sable timeout\n",
12876 vsi->seid, pf_q, (enable ? "en" : "dis"));
12883 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
12885 * @queue_pair: queue_pair
12887 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
12889 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12890 struct i40e_pf *pf = vsi->back;
12891 struct i40e_hw *hw = &pf->hw;
12893 /* All rings in a qp belong to the same qvector. */
12894 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
12895 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
12897 i40e_irq_dynamic_enable_icr0(pf);
12903 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
12905 * @queue_pair: queue_pair
12907 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
12909 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12910 struct i40e_pf *pf = vsi->back;
12911 struct i40e_hw *hw = &pf->hw;
12913 /* For simplicity, instead of removing the qp interrupt causes
12914 * from the interrupt linked list, we simply disable the interrupt, and
12915 * leave the list intact.
12917 * All rings in a qp belong to the same qvector.
12919 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12920 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
12922 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
12924 synchronize_irq(pf->msix_entries[intpf].vector);
12926 /* Legacy and MSI mode - this stops all interrupt handling */
12927 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
12928 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
12930 synchronize_irq(pf->pdev->irq);
12935 * i40e_queue_pair_disable - Disables a queue pair
12937 * @queue_pair: queue pair
12939 * Returns 0 on success, <0 on failure.
12941 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
12945 err = i40e_enter_busy_conf(vsi);
12949 i40e_queue_pair_disable_irq(vsi, queue_pair);
12950 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
12951 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
12952 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
12953 i40e_queue_pair_clean_rings(vsi, queue_pair);
12954 i40e_queue_pair_reset_stats(vsi, queue_pair);
12960 * i40e_queue_pair_enable - Enables a queue pair
12962 * @queue_pair: queue pair
12964 * Returns 0 on success, <0 on failure.
12966 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
12970 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
12974 if (i40e_enabled_xdp_vsi(vsi)) {
12975 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
12980 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
12984 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
12985 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
12986 i40e_queue_pair_enable_irq(vsi, queue_pair);
12988 i40e_exit_busy_conf(vsi);
12994 * i40e_xdp - implements ndo_bpf for i40e
12996 * @xdp: XDP command
12998 static int i40e_xdp(struct net_device *dev,
12999 struct netdev_bpf *xdp)
13001 struct i40e_netdev_priv *np = netdev_priv(dev);
13002 struct i40e_vsi *vsi = np->vsi;
13004 if (vsi->type != I40E_VSI_MAIN)
13007 switch (xdp->command) {
13008 case XDP_SETUP_PROG:
13009 return i40e_xdp_setup(vsi, xdp->prog);
13010 case XDP_SETUP_XSK_POOL:
13011 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13012 xdp->xsk.queue_id);
13018 static const struct net_device_ops i40e_netdev_ops = {
13019 .ndo_open = i40e_open,
13020 .ndo_stop = i40e_close,
13021 .ndo_start_xmit = i40e_lan_xmit_frame,
13022 .ndo_get_stats64 = i40e_get_netdev_stats_struct,
13023 .ndo_set_rx_mode = i40e_set_rx_mode,
13024 .ndo_validate_addr = eth_validate_addr,
13025 .ndo_set_mac_address = i40e_set_mac,
13026 .ndo_change_mtu = i40e_change_mtu,
13027 .ndo_do_ioctl = i40e_ioctl,
13028 .ndo_tx_timeout = i40e_tx_timeout,
13029 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
13030 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
13031 #ifdef CONFIG_NET_POLL_CONTROLLER
13032 .ndo_poll_controller = i40e_netpoll,
13034 .ndo_setup_tc = __i40e_setup_tc,
13035 .ndo_select_queue = i40e_lan_select_queue,
13036 .ndo_set_features = i40e_set_features,
13037 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
13038 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
13039 .ndo_get_vf_stats = i40e_get_vf_stats,
13040 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
13041 .ndo_get_vf_config = i40e_ndo_get_vf_config,
13042 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
13043 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
13044 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,
13045 .ndo_udp_tunnel_add = udp_tunnel_nic_add_port,
13046 .ndo_udp_tunnel_del = udp_tunnel_nic_del_port,
13047 .ndo_get_phys_port_id = i40e_get_phys_port_id,
13048 .ndo_fdb_add = i40e_ndo_fdb_add,
13049 .ndo_features_check = i40e_features_check,
13050 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,
13051 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,
13052 .ndo_bpf = i40e_xdp,
13053 .ndo_xdp_xmit = i40e_xdp_xmit,
13054 .ndo_xsk_wakeup = i40e_xsk_wakeup,
13055 .ndo_dfwd_add_station = i40e_fwd_add,
13056 .ndo_dfwd_del_station = i40e_fwd_del,
13060 * i40e_config_netdev - Setup the netdev flags
13061 * @vsi: the VSI being configured
13063 * Returns 0 on success, negative value on failure
13065 static int i40e_config_netdev(struct i40e_vsi *vsi)
13067 struct i40e_pf *pf = vsi->back;
13068 struct i40e_hw *hw = &pf->hw;
13069 struct i40e_netdev_priv *np;
13070 struct net_device *netdev;
13071 u8 broadcast[ETH_ALEN];
13072 u8 mac_addr[ETH_ALEN];
13074 netdev_features_t hw_enc_features;
13075 netdev_features_t hw_features;
13077 etherdev_size = sizeof(struct i40e_netdev_priv);
13078 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13082 vsi->netdev = netdev;
13083 np = netdev_priv(netdev);
13086 hw_enc_features = NETIF_F_SG |
13088 NETIF_F_IPV6_CSUM |
13090 NETIF_F_SOFT_FEATURES |
13095 NETIF_F_GSO_GRE_CSUM |
13096 NETIF_F_GSO_PARTIAL |
13097 NETIF_F_GSO_IPXIP4 |
13098 NETIF_F_GSO_IPXIP6 |
13099 NETIF_F_GSO_UDP_TUNNEL |
13100 NETIF_F_GSO_UDP_TUNNEL_CSUM |
13101 NETIF_F_GSO_UDP_L4 |
13107 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13108 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13110 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13112 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13114 netdev->hw_enc_features |= hw_enc_features;
13116 /* record features VLANs can make use of */
13117 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13119 /* enable macvlan offloads */
13120 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13122 hw_features = hw_enc_features |
13123 NETIF_F_HW_VLAN_CTAG_TX |
13124 NETIF_F_HW_VLAN_CTAG_RX;
13126 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13127 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13129 netdev->hw_features |= hw_features;
13131 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13132 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13134 if (vsi->type == I40E_VSI_MAIN) {
13135 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13136 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13137 /* The following steps are necessary for two reasons. First,
13138 * some older NVM configurations load a default MAC-VLAN
13139 * filter that will accept any tagged packet, and we want to
13140 * replace this with a normal filter. Additionally, it is
13141 * possible our MAC address was provided by the platform using
13142 * Open Firmware or similar.
13144 * Thus, we need to remove the default filter and install one
13145 * specific to the MAC address.
13147 i40e_rm_default_mac_filter(vsi, mac_addr);
13148 spin_lock_bh(&vsi->mac_filter_hash_lock);
13149 i40e_add_mac_filter(vsi, mac_addr);
13150 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13152 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13153 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13154 * the end, which is 4 bytes long, so force truncation of the
13155 * original name by IFNAMSIZ - 4
13157 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13159 pf->vsi[pf->lan_vsi]->netdev->name);
13160 eth_random_addr(mac_addr);
13162 spin_lock_bh(&vsi->mac_filter_hash_lock);
13163 i40e_add_mac_filter(vsi, mac_addr);
13164 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13167 /* Add the broadcast filter so that we initially will receive
13168 * broadcast packets. Note that when a new VLAN is first added the
13169 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13170 * specific filters as part of transitioning into "vlan" operation.
13171 * When more VLANs are added, the driver will copy each existing MAC
13172 * filter and add it for the new VLAN.
13174 * Broadcast filters are handled specially by
13175 * i40e_sync_filters_subtask, as the driver must to set the broadcast
13176 * promiscuous bit instead of adding this directly as a MAC/VLAN
13177 * filter. The subtask will update the correct broadcast promiscuous
13178 * bits as VLANs become active or inactive.
13180 eth_broadcast_addr(broadcast);
13181 spin_lock_bh(&vsi->mac_filter_hash_lock);
13182 i40e_add_mac_filter(vsi, broadcast);
13183 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13185 ether_addr_copy(netdev->dev_addr, mac_addr);
13186 ether_addr_copy(netdev->perm_addr, mac_addr);
13188 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13189 netdev->neigh_priv_len = sizeof(u32) * 4;
13191 netdev->priv_flags |= IFF_UNICAST_FLT;
13192 netdev->priv_flags |= IFF_SUPP_NOFCS;
13193 /* Setup netdev TC information */
13194 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13196 netdev->netdev_ops = &i40e_netdev_ops;
13197 netdev->watchdog_timeo = 5 * HZ;
13198 i40e_set_ethtool_ops(netdev);
13200 /* MTU range: 68 - 9706 */
13201 netdev->min_mtu = ETH_MIN_MTU;
13202 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13208 * i40e_vsi_delete - Delete a VSI from the switch
13209 * @vsi: the VSI being removed
13211 * Returns 0 on success, negative value on failure
13213 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13215 /* remove default VSI is not allowed */
13216 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13219 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13223 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13224 * @vsi: the VSI being queried
13226 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13228 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13230 struct i40e_veb *veb;
13231 struct i40e_pf *pf = vsi->back;
13233 /* Uplink is not a bridge so default to VEB */
13234 if (vsi->veb_idx >= I40E_MAX_VEB)
13237 veb = pf->veb[vsi->veb_idx];
13239 dev_info(&pf->pdev->dev,
13240 "There is no veb associated with the bridge\n");
13244 /* Uplink is a bridge in VEPA mode */
13245 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13248 /* Uplink is a bridge in VEB mode */
13252 /* VEPA is now default bridge, so return 0 */
13257 * i40e_add_vsi - Add a VSI to the switch
13258 * @vsi: the VSI being configured
13260 * This initializes a VSI context depending on the VSI type to be added and
13261 * passes it down to the add_vsi aq command.
13263 static int i40e_add_vsi(struct i40e_vsi *vsi)
13266 struct i40e_pf *pf = vsi->back;
13267 struct i40e_hw *hw = &pf->hw;
13268 struct i40e_vsi_context ctxt;
13269 struct i40e_mac_filter *f;
13270 struct hlist_node *h;
13273 u8 enabled_tc = 0x1; /* TC0 enabled */
13276 memset(&ctxt, 0, sizeof(ctxt));
13277 switch (vsi->type) {
13278 case I40E_VSI_MAIN:
13279 /* The PF's main VSI is already setup as part of the
13280 * device initialization, so we'll not bother with
13281 * the add_vsi call, but we will retrieve the current
13284 ctxt.seid = pf->main_vsi_seid;
13285 ctxt.pf_num = pf->hw.pf_id;
13287 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13288 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13290 dev_info(&pf->pdev->dev,
13291 "couldn't get PF vsi config, err %s aq_err %s\n",
13292 i40e_stat_str(&pf->hw, ret),
13293 i40e_aq_str(&pf->hw,
13294 pf->hw.aq.asq_last_status));
13297 vsi->info = ctxt.info;
13298 vsi->info.valid_sections = 0;
13300 vsi->seid = ctxt.seid;
13301 vsi->id = ctxt.vsi_number;
13303 enabled_tc = i40e_pf_get_tc_map(pf);
13305 /* Source pruning is enabled by default, so the flag is
13306 * negative logic - if it's set, we need to fiddle with
13307 * the VSI to disable source pruning.
13309 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13310 memset(&ctxt, 0, sizeof(ctxt));
13311 ctxt.seid = pf->main_vsi_seid;
13312 ctxt.pf_num = pf->hw.pf_id;
13314 ctxt.info.valid_sections |=
13315 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13316 ctxt.info.switch_id =
13317 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13318 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13320 dev_info(&pf->pdev->dev,
13321 "update vsi failed, err %s aq_err %s\n",
13322 i40e_stat_str(&pf->hw, ret),
13323 i40e_aq_str(&pf->hw,
13324 pf->hw.aq.asq_last_status));
13330 /* MFP mode setup queue map and update VSI */
13331 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13332 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13333 memset(&ctxt, 0, sizeof(ctxt));
13334 ctxt.seid = pf->main_vsi_seid;
13335 ctxt.pf_num = pf->hw.pf_id;
13337 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13338 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13340 dev_info(&pf->pdev->dev,
13341 "update vsi failed, err %s aq_err %s\n",
13342 i40e_stat_str(&pf->hw, ret),
13343 i40e_aq_str(&pf->hw,
13344 pf->hw.aq.asq_last_status));
13348 /* update the local VSI info queue map */
13349 i40e_vsi_update_queue_map(vsi, &ctxt);
13350 vsi->info.valid_sections = 0;
13352 /* Default/Main VSI is only enabled for TC0
13353 * reconfigure it to enable all TCs that are
13354 * available on the port in SFP mode.
13355 * For MFP case the iSCSI PF would use this
13356 * flow to enable LAN+iSCSI TC.
13358 ret = i40e_vsi_config_tc(vsi, enabled_tc);
13360 /* Single TC condition is not fatal,
13361 * message and continue
13363 dev_info(&pf->pdev->dev,
13364 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13366 i40e_stat_str(&pf->hw, ret),
13367 i40e_aq_str(&pf->hw,
13368 pf->hw.aq.asq_last_status));
13373 case I40E_VSI_FDIR:
13374 ctxt.pf_num = hw->pf_id;
13376 ctxt.uplink_seid = vsi->uplink_seid;
13377 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13378 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13379 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13380 (i40e_is_vsi_uplink_mode_veb(vsi))) {
13381 ctxt.info.valid_sections |=
13382 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13383 ctxt.info.switch_id =
13384 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13386 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13389 case I40E_VSI_VMDQ2:
13390 ctxt.pf_num = hw->pf_id;
13392 ctxt.uplink_seid = vsi->uplink_seid;
13393 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13394 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13396 /* This VSI is connected to VEB so the switch_id
13397 * should be set to zero by default.
13399 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13400 ctxt.info.valid_sections |=
13401 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13402 ctxt.info.switch_id =
13403 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13406 /* Setup the VSI tx/rx queue map for TC0 only for now */
13407 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13410 case I40E_VSI_SRIOV:
13411 ctxt.pf_num = hw->pf_id;
13412 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13413 ctxt.uplink_seid = vsi->uplink_seid;
13414 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13415 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13417 /* This VSI is connected to VEB so the switch_id
13418 * should be set to zero by default.
13420 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13421 ctxt.info.valid_sections |=
13422 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13423 ctxt.info.switch_id =
13424 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13427 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13428 ctxt.info.valid_sections |=
13429 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13430 ctxt.info.queueing_opt_flags |=
13431 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13432 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13435 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13436 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13437 if (pf->vf[vsi->vf_id].spoofchk) {
13438 ctxt.info.valid_sections |=
13439 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13440 ctxt.info.sec_flags |=
13441 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13442 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13444 /* Setup the VSI tx/rx queue map for TC0 only for now */
13445 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13448 case I40E_VSI_IWARP:
13449 /* send down message to iWARP */
13456 if (vsi->type != I40E_VSI_MAIN) {
13457 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13459 dev_info(&vsi->back->pdev->dev,
13460 "add vsi failed, err %s aq_err %s\n",
13461 i40e_stat_str(&pf->hw, ret),
13462 i40e_aq_str(&pf->hw,
13463 pf->hw.aq.asq_last_status));
13467 vsi->info = ctxt.info;
13468 vsi->info.valid_sections = 0;
13469 vsi->seid = ctxt.seid;
13470 vsi->id = ctxt.vsi_number;
13473 spin_lock_bh(&vsi->mac_filter_hash_lock);
13474 vsi->active_filters = 0;
13475 /* If macvlan filters already exist, force them to get loaded */
13476 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13477 f->state = I40E_FILTER_NEW;
13480 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13481 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13484 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13485 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13488 /* Update VSI BW information */
13489 ret = i40e_vsi_get_bw_info(vsi);
13491 dev_info(&pf->pdev->dev,
13492 "couldn't get vsi bw info, err %s aq_err %s\n",
13493 i40e_stat_str(&pf->hw, ret),
13494 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13495 /* VSI is already added so not tearing that up */
13504 * i40e_vsi_release - Delete a VSI and free its resources
13505 * @vsi: the VSI being removed
13507 * Returns 0 on success or < 0 on error
13509 int i40e_vsi_release(struct i40e_vsi *vsi)
13511 struct i40e_mac_filter *f;
13512 struct hlist_node *h;
13513 struct i40e_veb *veb = NULL;
13514 struct i40e_pf *pf;
13520 /* release of a VEB-owner or last VSI is not allowed */
13521 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13522 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13523 vsi->seid, vsi->uplink_seid);
13526 if (vsi == pf->vsi[pf->lan_vsi] &&
13527 !test_bit(__I40E_DOWN, pf->state)) {
13528 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13531 set_bit(__I40E_VSI_RELEASING, vsi->state);
13532 uplink_seid = vsi->uplink_seid;
13533 if (vsi->type != I40E_VSI_SRIOV) {
13534 if (vsi->netdev_registered) {
13535 vsi->netdev_registered = false;
13537 /* results in a call to i40e_close() */
13538 unregister_netdev(vsi->netdev);
13541 i40e_vsi_close(vsi);
13543 i40e_vsi_disable_irq(vsi);
13546 spin_lock_bh(&vsi->mac_filter_hash_lock);
13548 /* clear the sync flag on all filters */
13550 __dev_uc_unsync(vsi->netdev, NULL);
13551 __dev_mc_unsync(vsi->netdev, NULL);
13554 /* make sure any remaining filters are marked for deletion */
13555 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13556 __i40e_del_filter(vsi, f);
13558 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13560 i40e_sync_vsi_filters(vsi);
13562 i40e_vsi_delete(vsi);
13563 i40e_vsi_free_q_vectors(vsi);
13565 free_netdev(vsi->netdev);
13566 vsi->netdev = NULL;
13568 i40e_vsi_clear_rings(vsi);
13569 i40e_vsi_clear(vsi);
13571 /* If this was the last thing on the VEB, except for the
13572 * controlling VSI, remove the VEB, which puts the controlling
13573 * VSI onto the next level down in the switch.
13575 * Well, okay, there's one more exception here: don't remove
13576 * the orphan VEBs yet. We'll wait for an explicit remove request
13577 * from up the network stack.
13579 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13581 pf->vsi[i]->uplink_seid == uplink_seid &&
13582 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13583 n++; /* count the VSIs */
13586 for (i = 0; i < I40E_MAX_VEB; i++) {
13589 if (pf->veb[i]->uplink_seid == uplink_seid)
13590 n++; /* count the VEBs */
13591 if (pf->veb[i]->seid == uplink_seid)
13594 if (n == 0 && veb && veb->uplink_seid != 0)
13595 i40e_veb_release(veb);
13601 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
13602 * @vsi: ptr to the VSI
13604 * This should only be called after i40e_vsi_mem_alloc() which allocates the
13605 * corresponding SW VSI structure and initializes num_queue_pairs for the
13606 * newly allocated VSI.
13608 * Returns 0 on success or negative on failure
13610 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13613 struct i40e_pf *pf = vsi->back;
13615 if (vsi->q_vectors[0]) {
13616 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13621 if (vsi->base_vector) {
13622 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13623 vsi->seid, vsi->base_vector);
13627 ret = i40e_vsi_alloc_q_vectors(vsi);
13629 dev_info(&pf->pdev->dev,
13630 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13631 vsi->num_q_vectors, vsi->seid, ret);
13632 vsi->num_q_vectors = 0;
13633 goto vector_setup_out;
13636 /* In Legacy mode, we do not have to get any other vector since we
13637 * piggyback on the misc/ICR0 for queue interrupts.
13639 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13641 if (vsi->num_q_vectors)
13642 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13643 vsi->num_q_vectors, vsi->idx);
13644 if (vsi->base_vector < 0) {
13645 dev_info(&pf->pdev->dev,
13646 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13647 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13648 i40e_vsi_free_q_vectors(vsi);
13650 goto vector_setup_out;
13658 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
13659 * @vsi: pointer to the vsi.
13661 * This re-allocates a vsi's queue resources.
13663 * Returns pointer to the successfully allocated and configured VSI sw struct
13664 * on success, otherwise returns NULL on failure.
13666 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13668 u16 alloc_queue_pairs;
13669 struct i40e_pf *pf;
13678 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
13679 i40e_vsi_clear_rings(vsi);
13681 i40e_vsi_free_arrays(vsi, false);
13682 i40e_set_num_rings_in_vsi(vsi);
13683 ret = i40e_vsi_alloc_arrays(vsi, false);
13687 alloc_queue_pairs = vsi->alloc_queue_pairs *
13688 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13690 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13692 dev_info(&pf->pdev->dev,
13693 "failed to get tracking for %d queues for VSI %d err %d\n",
13694 alloc_queue_pairs, vsi->seid, ret);
13697 vsi->base_queue = ret;
13699 /* Update the FW view of the VSI. Force a reset of TC and queue
13700 * layout configurations.
13702 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13703 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13704 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13705 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13706 if (vsi->type == I40E_VSI_MAIN)
13707 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
13709 /* assign it some queues */
13710 ret = i40e_alloc_rings(vsi);
13714 /* map all of the rings to the q_vectors */
13715 i40e_vsi_map_rings_to_vectors(vsi);
13719 i40e_vsi_free_q_vectors(vsi);
13720 if (vsi->netdev_registered) {
13721 vsi->netdev_registered = false;
13722 unregister_netdev(vsi->netdev);
13723 free_netdev(vsi->netdev);
13724 vsi->netdev = NULL;
13726 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13728 i40e_vsi_clear(vsi);
13733 * i40e_vsi_setup - Set up a VSI by a given type
13734 * @pf: board private structure
13736 * @uplink_seid: the switch element to link to
13737 * @param1: usage depends upon VSI type. For VF types, indicates VF id
13739 * This allocates the sw VSI structure and its queue resources, then add a VSI
13740 * to the identified VEB.
13742 * Returns pointer to the successfully allocated and configure VSI sw struct on
13743 * success, otherwise returns NULL on failure.
13745 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
13746 u16 uplink_seid, u32 param1)
13748 struct i40e_vsi *vsi = NULL;
13749 struct i40e_veb *veb = NULL;
13750 u16 alloc_queue_pairs;
13754 /* The requested uplink_seid must be either
13755 * - the PF's port seid
13756 * no VEB is needed because this is the PF
13757 * or this is a Flow Director special case VSI
13758 * - seid of an existing VEB
13759 * - seid of a VSI that owns an existing VEB
13760 * - seid of a VSI that doesn't own a VEB
13761 * a new VEB is created and the VSI becomes the owner
13762 * - seid of the PF VSI, which is what creates the first VEB
13763 * this is a special case of the previous
13765 * Find which uplink_seid we were given and create a new VEB if needed
13767 for (i = 0; i < I40E_MAX_VEB; i++) {
13768 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
13774 if (!veb && uplink_seid != pf->mac_seid) {
13776 for (i = 0; i < pf->num_alloc_vsi; i++) {
13777 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
13783 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
13788 if (vsi->uplink_seid == pf->mac_seid)
13789 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
13790 vsi->tc_config.enabled_tc);
13791 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
13792 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13793 vsi->tc_config.enabled_tc);
13795 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
13796 dev_info(&vsi->back->pdev->dev,
13797 "New VSI creation error, uplink seid of LAN VSI expected.\n");
13800 /* We come up by default in VEPA mode if SRIOV is not
13801 * already enabled, in which case we can't force VEPA
13804 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
13805 veb->bridge_mode = BRIDGE_MODE_VEPA;
13806 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
13808 i40e_config_bridge_mode(veb);
13810 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13811 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13815 dev_info(&pf->pdev->dev, "couldn't add VEB\n");
13819 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
13820 uplink_seid = veb->seid;
13823 /* get vsi sw struct */
13824 v_idx = i40e_vsi_mem_alloc(pf, type);
13827 vsi = pf->vsi[v_idx];
13831 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
13833 if (type == I40E_VSI_MAIN)
13834 pf->lan_vsi = v_idx;
13835 else if (type == I40E_VSI_SRIOV)
13836 vsi->vf_id = param1;
13837 /* assign it some queues */
13838 alloc_queue_pairs = vsi->alloc_queue_pairs *
13839 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13841 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13843 dev_info(&pf->pdev->dev,
13844 "failed to get tracking for %d queues for VSI %d err=%d\n",
13845 alloc_queue_pairs, vsi->seid, ret);
13848 vsi->base_queue = ret;
13850 /* get a VSI from the hardware */
13851 vsi->uplink_seid = uplink_seid;
13852 ret = i40e_add_vsi(vsi);
13856 switch (vsi->type) {
13857 /* setup the netdev if needed */
13858 case I40E_VSI_MAIN:
13859 case I40E_VSI_VMDQ2:
13860 ret = i40e_config_netdev(vsi);
13863 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
13866 ret = register_netdev(vsi->netdev);
13869 vsi->netdev_registered = true;
13870 netif_carrier_off(vsi->netdev);
13871 #ifdef CONFIG_I40E_DCB
13872 /* Setup DCB netlink interface */
13873 i40e_dcbnl_setup(vsi);
13874 #endif /* CONFIG_I40E_DCB */
13876 case I40E_VSI_FDIR:
13877 /* set up vectors and rings if needed */
13878 ret = i40e_vsi_setup_vectors(vsi);
13882 ret = i40e_alloc_rings(vsi);
13886 /* map all of the rings to the q_vectors */
13887 i40e_vsi_map_rings_to_vectors(vsi);
13889 i40e_vsi_reset_stats(vsi);
13892 /* no netdev or rings for the other VSI types */
13896 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
13897 (vsi->type == I40E_VSI_VMDQ2)) {
13898 ret = i40e_vsi_config_rss(vsi);
13903 i40e_vsi_free_q_vectors(vsi);
13905 if (vsi->netdev_registered) {
13906 vsi->netdev_registered = false;
13907 unregister_netdev(vsi->netdev);
13908 free_netdev(vsi->netdev);
13909 vsi->netdev = NULL;
13912 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13914 i40e_vsi_clear(vsi);
13920 * i40e_veb_get_bw_info - Query VEB BW information
13921 * @veb: the veb to query
13923 * Query the Tx scheduler BW configuration data for given VEB
13925 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
13927 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
13928 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
13929 struct i40e_pf *pf = veb->pf;
13930 struct i40e_hw *hw = &pf->hw;
13935 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
13938 dev_info(&pf->pdev->dev,
13939 "query veb bw config failed, err %s aq_err %s\n",
13940 i40e_stat_str(&pf->hw, ret),
13941 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13945 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
13948 dev_info(&pf->pdev->dev,
13949 "query veb bw ets config failed, err %s aq_err %s\n",
13950 i40e_stat_str(&pf->hw, ret),
13951 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13955 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
13956 veb->bw_max_quanta = ets_data.tc_bw_max;
13957 veb->is_abs_credits = bw_data.absolute_credits_enable;
13958 veb->enabled_tc = ets_data.tc_valid_bits;
13959 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
13960 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
13961 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
13962 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
13963 veb->bw_tc_limit_credits[i] =
13964 le16_to_cpu(bw_data.tc_bw_limits[i]);
13965 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
13973 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
13974 * @pf: board private structure
13976 * On error: returns error code (negative)
13977 * On success: returns vsi index in PF (positive)
13979 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
13982 struct i40e_veb *veb;
13985 /* Need to protect the allocation of switch elements at the PF level */
13986 mutex_lock(&pf->switch_mutex);
13988 /* VEB list may be fragmented if VEB creation/destruction has
13989 * been happening. We can afford to do a quick scan to look
13990 * for any free slots in the list.
13992 * find next empty veb slot, looping back around if necessary
13995 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
13997 if (i >= I40E_MAX_VEB) {
13999 goto err_alloc_veb; /* out of VEB slots! */
14002 veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14005 goto err_alloc_veb;
14009 veb->enabled_tc = 1;
14014 mutex_unlock(&pf->switch_mutex);
14019 * i40e_switch_branch_release - Delete a branch of the switch tree
14020 * @branch: where to start deleting
14022 * This uses recursion to find the tips of the branch to be
14023 * removed, deleting until we get back to and can delete this VEB.
14025 static void i40e_switch_branch_release(struct i40e_veb *branch)
14027 struct i40e_pf *pf = branch->pf;
14028 u16 branch_seid = branch->seid;
14029 u16 veb_idx = branch->idx;
14032 /* release any VEBs on this VEB - RECURSION */
14033 for (i = 0; i < I40E_MAX_VEB; i++) {
14036 if (pf->veb[i]->uplink_seid == branch->seid)
14037 i40e_switch_branch_release(pf->veb[i]);
14040 /* Release the VSIs on this VEB, but not the owner VSI.
14042 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14043 * the VEB itself, so don't use (*branch) after this loop.
14045 for (i = 0; i < pf->num_alloc_vsi; i++) {
14048 if (pf->vsi[i]->uplink_seid == branch_seid &&
14049 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14050 i40e_vsi_release(pf->vsi[i]);
14054 /* There's one corner case where the VEB might not have been
14055 * removed, so double check it here and remove it if needed.
14056 * This case happens if the veb was created from the debugfs
14057 * commands and no VSIs were added to it.
14059 if (pf->veb[veb_idx])
14060 i40e_veb_release(pf->veb[veb_idx]);
14064 * i40e_veb_clear - remove veb struct
14065 * @veb: the veb to remove
14067 static void i40e_veb_clear(struct i40e_veb *veb)
14073 struct i40e_pf *pf = veb->pf;
14075 mutex_lock(&pf->switch_mutex);
14076 if (pf->veb[veb->idx] == veb)
14077 pf->veb[veb->idx] = NULL;
14078 mutex_unlock(&pf->switch_mutex);
14085 * i40e_veb_release - Delete a VEB and free its resources
14086 * @veb: the VEB being removed
14088 void i40e_veb_release(struct i40e_veb *veb)
14090 struct i40e_vsi *vsi = NULL;
14091 struct i40e_pf *pf;
14096 /* find the remaining VSI and check for extras */
14097 for (i = 0; i < pf->num_alloc_vsi; i++) {
14098 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14104 dev_info(&pf->pdev->dev,
14105 "can't remove VEB %d with %d VSIs left\n",
14110 /* move the remaining VSI to uplink veb */
14111 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14112 if (veb->uplink_seid) {
14113 vsi->uplink_seid = veb->uplink_seid;
14114 if (veb->uplink_seid == pf->mac_seid)
14115 vsi->veb_idx = I40E_NO_VEB;
14117 vsi->veb_idx = veb->veb_idx;
14120 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14121 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14124 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14125 i40e_veb_clear(veb);
14129 * i40e_add_veb - create the VEB in the switch
14130 * @veb: the VEB to be instantiated
14131 * @vsi: the controlling VSI
14133 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14135 struct i40e_pf *pf = veb->pf;
14136 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14139 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14140 veb->enabled_tc, false,
14141 &veb->seid, enable_stats, NULL);
14143 /* get a VEB from the hardware */
14145 dev_info(&pf->pdev->dev,
14146 "couldn't add VEB, err %s aq_err %s\n",
14147 i40e_stat_str(&pf->hw, ret),
14148 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14152 /* get statistics counter */
14153 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14154 &veb->stats_idx, NULL, NULL, NULL);
14156 dev_info(&pf->pdev->dev,
14157 "couldn't get VEB statistics idx, err %s aq_err %s\n",
14158 i40e_stat_str(&pf->hw, ret),
14159 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14162 ret = i40e_veb_get_bw_info(veb);
14164 dev_info(&pf->pdev->dev,
14165 "couldn't get VEB bw info, err %s aq_err %s\n",
14166 i40e_stat_str(&pf->hw, ret),
14167 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14168 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14172 vsi->uplink_seid = veb->seid;
14173 vsi->veb_idx = veb->idx;
14174 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14180 * i40e_veb_setup - Set up a VEB
14181 * @pf: board private structure
14182 * @flags: VEB setup flags
14183 * @uplink_seid: the switch element to link to
14184 * @vsi_seid: the initial VSI seid
14185 * @enabled_tc: Enabled TC bit-map
14187 * This allocates the sw VEB structure and links it into the switch
14188 * It is possible and legal for this to be a duplicate of an already
14189 * existing VEB. It is also possible for both uplink and vsi seids
14190 * to be zero, in order to create a floating VEB.
14192 * Returns pointer to the successfully allocated VEB sw struct on
14193 * success, otherwise returns NULL on failure.
14195 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14196 u16 uplink_seid, u16 vsi_seid,
14199 struct i40e_veb *veb, *uplink_veb = NULL;
14200 int vsi_idx, veb_idx;
14203 /* if one seid is 0, the other must be 0 to create a floating relay */
14204 if ((uplink_seid == 0 || vsi_seid == 0) &&
14205 (uplink_seid + vsi_seid != 0)) {
14206 dev_info(&pf->pdev->dev,
14207 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14208 uplink_seid, vsi_seid);
14212 /* make sure there is such a vsi and uplink */
14213 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14214 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14216 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14217 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14222 if (uplink_seid && uplink_seid != pf->mac_seid) {
14223 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14224 if (pf->veb[veb_idx] &&
14225 pf->veb[veb_idx]->seid == uplink_seid) {
14226 uplink_veb = pf->veb[veb_idx];
14231 dev_info(&pf->pdev->dev,
14232 "uplink seid %d not found\n", uplink_seid);
14237 /* get veb sw struct */
14238 veb_idx = i40e_veb_mem_alloc(pf);
14241 veb = pf->veb[veb_idx];
14242 veb->flags = flags;
14243 veb->uplink_seid = uplink_seid;
14244 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14245 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14247 /* create the VEB in the switch */
14248 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14251 if (vsi_idx == pf->lan_vsi)
14252 pf->lan_veb = veb->idx;
14257 i40e_veb_clear(veb);
14263 * i40e_setup_pf_switch_element - set PF vars based on switch type
14264 * @pf: board private structure
14265 * @ele: element we are building info from
14266 * @num_reported: total number of elements
14267 * @printconfig: should we print the contents
14269 * helper function to assist in extracting a few useful SEID values.
14271 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14272 struct i40e_aqc_switch_config_element_resp *ele,
14273 u16 num_reported, bool printconfig)
14275 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14276 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14277 u8 element_type = ele->element_type;
14278 u16 seid = le16_to_cpu(ele->seid);
14281 dev_info(&pf->pdev->dev,
14282 "type=%d seid=%d uplink=%d downlink=%d\n",
14283 element_type, seid, uplink_seid, downlink_seid);
14285 switch (element_type) {
14286 case I40E_SWITCH_ELEMENT_TYPE_MAC:
14287 pf->mac_seid = seid;
14289 case I40E_SWITCH_ELEMENT_TYPE_VEB:
14291 if (uplink_seid != pf->mac_seid)
14293 if (pf->lan_veb >= I40E_MAX_VEB) {
14296 /* find existing or else empty VEB */
14297 for (v = 0; v < I40E_MAX_VEB; v++) {
14298 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14303 if (pf->lan_veb >= I40E_MAX_VEB) {
14304 v = i40e_veb_mem_alloc(pf);
14310 if (pf->lan_veb >= I40E_MAX_VEB)
14313 pf->veb[pf->lan_veb]->seid = seid;
14314 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14315 pf->veb[pf->lan_veb]->pf = pf;
14316 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14318 case I40E_SWITCH_ELEMENT_TYPE_VSI:
14319 if (num_reported != 1)
14321 /* This is immediately after a reset so we can assume this is
14324 pf->mac_seid = uplink_seid;
14325 pf->pf_seid = downlink_seid;
14326 pf->main_vsi_seid = seid;
14328 dev_info(&pf->pdev->dev,
14329 "pf_seid=%d main_vsi_seid=%d\n",
14330 pf->pf_seid, pf->main_vsi_seid);
14332 case I40E_SWITCH_ELEMENT_TYPE_PF:
14333 case I40E_SWITCH_ELEMENT_TYPE_VF:
14334 case I40E_SWITCH_ELEMENT_TYPE_EMP:
14335 case I40E_SWITCH_ELEMENT_TYPE_BMC:
14336 case I40E_SWITCH_ELEMENT_TYPE_PE:
14337 case I40E_SWITCH_ELEMENT_TYPE_PA:
14338 /* ignore these for now */
14341 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14342 element_type, seid);
14348 * i40e_fetch_switch_configuration - Get switch config from firmware
14349 * @pf: board private structure
14350 * @printconfig: should we print the contents
14352 * Get the current switch configuration from the device and
14353 * extract a few useful SEID values.
14355 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14357 struct i40e_aqc_get_switch_config_resp *sw_config;
14363 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14367 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14369 u16 num_reported, num_total;
14371 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14375 dev_info(&pf->pdev->dev,
14376 "get switch config failed err %s aq_err %s\n",
14377 i40e_stat_str(&pf->hw, ret),
14378 i40e_aq_str(&pf->hw,
14379 pf->hw.aq.asq_last_status));
14384 num_reported = le16_to_cpu(sw_config->header.num_reported);
14385 num_total = le16_to_cpu(sw_config->header.num_total);
14388 dev_info(&pf->pdev->dev,
14389 "header: %d reported %d total\n",
14390 num_reported, num_total);
14392 for (i = 0; i < num_reported; i++) {
14393 struct i40e_aqc_switch_config_element_resp *ele =
14394 &sw_config->element[i];
14396 i40e_setup_pf_switch_element(pf, ele, num_reported,
14399 } while (next_seid != 0);
14406 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14407 * @pf: board private structure
14408 * @reinit: if the Main VSI needs to re-initialized.
14409 * @lock_acquired: indicates whether or not the lock has been acquired
14411 * Returns 0 on success, negative value on failure
14413 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
14418 /* find out what's out there already */
14419 ret = i40e_fetch_switch_configuration(pf, false);
14421 dev_info(&pf->pdev->dev,
14422 "couldn't fetch switch config, err %s aq_err %s\n",
14423 i40e_stat_str(&pf->hw, ret),
14424 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14427 i40e_pf_reset_stats(pf);
14429 /* set the switch config bit for the whole device to
14430 * support limited promisc or true promisc
14431 * when user requests promisc. The default is limited
14435 if ((pf->hw.pf_id == 0) &&
14436 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14437 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14438 pf->last_sw_conf_flags = flags;
14441 if (pf->hw.pf_id == 0) {
14444 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14445 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14447 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14448 dev_info(&pf->pdev->dev,
14449 "couldn't set switch config bits, err %s aq_err %s\n",
14450 i40e_stat_str(&pf->hw, ret),
14451 i40e_aq_str(&pf->hw,
14452 pf->hw.aq.asq_last_status));
14453 /* not a fatal problem, just keep going */
14455 pf->last_sw_conf_valid_flags = valid_flags;
14458 /* first time setup */
14459 if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14460 struct i40e_vsi *vsi = NULL;
14463 /* Set up the PF VSI associated with the PF's main VSI
14464 * that is already in the HW switch
14466 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14467 uplink_seid = pf->veb[pf->lan_veb]->seid;
14469 uplink_seid = pf->mac_seid;
14470 if (pf->lan_vsi == I40E_NO_VSI)
14471 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14473 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14475 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14476 i40e_cloud_filter_exit(pf);
14477 i40e_fdir_teardown(pf);
14481 /* force a reset of TC and queue layout configurations */
14482 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14484 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14485 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14486 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14488 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14490 i40e_fdir_sb_setup(pf);
14492 /* Setup static PF queue filter control settings */
14493 ret = i40e_setup_pf_filter_control(pf);
14495 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14497 /* Failure here should not stop continuing other steps */
14500 /* enable RSS in the HW, even for only one queue, as the stack can use
14503 if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14504 i40e_pf_config_rss(pf);
14506 /* fill in link information and enable LSE reporting */
14507 i40e_link_event(pf);
14509 /* Initialize user-specific link properties */
14510 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14511 I40E_AQ_AN_COMPLETED) ? true : false);
14515 if (!lock_acquired)
14518 /* repopulate tunnel port filters */
14519 udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
14521 if (!lock_acquired)
14528 * i40e_determine_queue_usage - Work out queue distribution
14529 * @pf: board private structure
14531 static void i40e_determine_queue_usage(struct i40e_pf *pf)
14536 pf->num_lan_qps = 0;
14538 /* Find the max queues to be put into basic use. We'll always be
14539 * using TC0, whether or not DCB is running, and TC0 will get the
14542 queues_left = pf->hw.func_caps.num_tx_qp;
14544 if ((queues_left == 1) ||
14545 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14546 /* one qp for PF, no queues for anything else */
14548 pf->alloc_rss_size = pf->num_lan_qps = 1;
14550 /* make sure all the fancies are disabled */
14551 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
14552 I40E_FLAG_IWARP_ENABLED |
14553 I40E_FLAG_FD_SB_ENABLED |
14554 I40E_FLAG_FD_ATR_ENABLED |
14555 I40E_FLAG_DCB_CAPABLE |
14556 I40E_FLAG_DCB_ENABLED |
14557 I40E_FLAG_SRIOV_ENABLED |
14558 I40E_FLAG_VMDQ_ENABLED);
14559 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14560 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14561 I40E_FLAG_FD_SB_ENABLED |
14562 I40E_FLAG_FD_ATR_ENABLED |
14563 I40E_FLAG_DCB_CAPABLE))) {
14564 /* one qp for PF */
14565 pf->alloc_rss_size = pf->num_lan_qps = 1;
14566 queues_left -= pf->num_lan_qps;
14568 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
14569 I40E_FLAG_IWARP_ENABLED |
14570 I40E_FLAG_FD_SB_ENABLED |
14571 I40E_FLAG_FD_ATR_ENABLED |
14572 I40E_FLAG_DCB_ENABLED |
14573 I40E_FLAG_VMDQ_ENABLED);
14574 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14576 /* Not enough queues for all TCs */
14577 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14578 (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14579 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14580 I40E_FLAG_DCB_ENABLED);
14581 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14584 /* limit lan qps to the smaller of qps, cpus or msix */
14585 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14586 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14587 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14588 pf->num_lan_qps = q_max;
14590 queues_left -= pf->num_lan_qps;
14593 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14594 if (queues_left > 1) {
14595 queues_left -= 1; /* save 1 queue for FD */
14597 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14598 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14599 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14603 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14604 pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14605 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14606 (queues_left / pf->num_vf_qps));
14607 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14610 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14611 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14612 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14613 (queues_left / pf->num_vmdq_qps));
14614 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14617 pf->queues_left = queues_left;
14618 dev_dbg(&pf->pdev->dev,
14619 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14620 pf->hw.func_caps.num_tx_qp,
14621 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14622 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14623 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14628 * i40e_setup_pf_filter_control - Setup PF static filter control
14629 * @pf: PF to be setup
14631 * i40e_setup_pf_filter_control sets up a PF's initial filter control
14632 * settings. If PE/FCoE are enabled then it will also set the per PF
14633 * based filter sizes required for them. It also enables Flow director,
14634 * ethertype and macvlan type filter settings for the pf.
14636 * Returns 0 on success, negative on failure
14638 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14640 struct i40e_filter_control_settings *settings = &pf->filter_settings;
14642 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14644 /* Flow Director is enabled */
14645 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14646 settings->enable_fdir = true;
14648 /* Ethtype and MACVLAN filters enabled for PF */
14649 settings->enable_ethtype = true;
14650 settings->enable_macvlan = true;
14652 if (i40e_set_filter_control(&pf->hw, settings))
14658 #define INFO_STRING_LEN 255
14659 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
14660 static void i40e_print_features(struct i40e_pf *pf)
14662 struct i40e_hw *hw = &pf->hw;
14666 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14670 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
14671 #ifdef CONFIG_PCI_IOV
14672 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
14674 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
14675 pf->hw.func_caps.num_vsis,
14676 pf->vsi[pf->lan_vsi]->num_queue_pairs);
14677 if (pf->flags & I40E_FLAG_RSS_ENABLED)
14678 i += scnprintf(&buf[i], REMAIN(i), " RSS");
14679 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
14680 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
14681 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14682 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
14683 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
14685 if (pf->flags & I40E_FLAG_DCB_CAPABLE)
14686 i += scnprintf(&buf[i], REMAIN(i), " DCB");
14687 i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
14688 i += scnprintf(&buf[i], REMAIN(i), " Geneve");
14689 if (pf->flags & I40E_FLAG_PTP)
14690 i += scnprintf(&buf[i], REMAIN(i), " PTP");
14691 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
14692 i += scnprintf(&buf[i], REMAIN(i), " VEB");
14694 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
14696 dev_info(&pf->pdev->dev, "%s\n", buf);
14698 WARN_ON(i > INFO_STRING_LEN);
14702 * i40e_get_platform_mac_addr - get platform-specific MAC address
14703 * @pdev: PCI device information struct
14704 * @pf: board private structure
14706 * Look up the MAC address for the device. First we'll try
14707 * eth_platform_get_mac_address, which will check Open Firmware, or arch
14708 * specific fallback. Otherwise, we'll default to the stored value in
14711 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
14713 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
14714 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
14718 * i40e_set_fec_in_flags - helper function for setting FEC options in flags
14719 * @fec_cfg: FEC option to set in flags
14720 * @flags: ptr to flags in which we set FEC option
14722 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
14724 if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
14725 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
14726 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
14727 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
14728 *flags |= I40E_FLAG_RS_FEC;
14729 *flags &= ~I40E_FLAG_BASE_R_FEC;
14731 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
14732 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
14733 *flags |= I40E_FLAG_BASE_R_FEC;
14734 *flags &= ~I40E_FLAG_RS_FEC;
14737 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
14741 * i40e_check_recovery_mode - check if we are running transition firmware
14742 * @pf: board private structure
14744 * Check registers indicating the firmware runs in recovery mode. Sets the
14745 * appropriate driver state.
14747 * Returns true if the recovery mode was detected, false otherwise
14749 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
14751 u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
14753 if (val & I40E_GL_FWSTS_FWS1B_MASK) {
14754 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
14755 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
14756 set_bit(__I40E_RECOVERY_MODE, pf->state);
14760 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
14761 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
14767 * i40e_pf_loop_reset - perform reset in a loop.
14768 * @pf: board private structure
14770 * This function is useful when a NIC is about to enter recovery mode.
14771 * When a NIC's internal data structures are corrupted the NIC's
14772 * firmware is going to enter recovery mode.
14773 * Right after a POR it takes about 7 minutes for firmware to enter
14774 * recovery mode. Until that time a NIC is in some kind of intermediate
14775 * state. After that time period the NIC almost surely enters
14776 * recovery mode. The only way for a driver to detect intermediate
14777 * state is to issue a series of pf-resets and check a return value.
14778 * If a PF reset returns success then the firmware could be in recovery
14779 * mode so the caller of this code needs to check for recovery mode
14780 * if this function returns success. There is a little chance that
14781 * firmware will hang in intermediate state forever.
14782 * Since waiting 7 minutes is quite a lot of time this function waits
14783 * 10 seconds and then gives up by returning an error.
14785 * Return 0 on success, negative on failure.
14787 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf)
14789 /* wait max 10 seconds for PF reset to succeed */
14790 const unsigned long time_end = jiffies + 10 * HZ;
14792 struct i40e_hw *hw = &pf->hw;
14795 ret = i40e_pf_reset(hw);
14796 while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
14797 usleep_range(10000, 20000);
14798 ret = i40e_pf_reset(hw);
14801 if (ret == I40E_SUCCESS)
14804 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
14810 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
14811 * @pf: board private structure
14813 * Check FW registers to determine if FW issued unexpected EMP Reset.
14814 * Every time when unexpected EMP Reset occurs the FW increments
14815 * a counter of unexpected EMP Resets. When the counter reaches 10
14816 * the FW should enter the Recovery mode
14818 * Returns true if FW issued unexpected EMP Reset
14820 static bool i40e_check_fw_empr(struct i40e_pf *pf)
14822 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
14823 I40E_GL_FWSTS_FWS1B_MASK;
14824 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
14825 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
14829 * i40e_handle_resets - handle EMP resets and PF resets
14830 * @pf: board private structure
14832 * Handle both EMP resets and PF resets and conclude whether there are
14833 * any issues regarding these resets. If there are any issues then
14834 * generate log entry.
14836 * Return 0 if NIC is healthy or negative value when there are issues
14839 static i40e_status i40e_handle_resets(struct i40e_pf *pf)
14841 const i40e_status pfr = i40e_pf_loop_reset(pf);
14842 const bool is_empr = i40e_check_fw_empr(pf);
14844 if (is_empr || pfr != I40E_SUCCESS)
14845 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
14847 return is_empr ? I40E_ERR_RESET_FAILED : pfr;
14851 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
14852 * @pf: board private structure
14853 * @hw: ptr to the hardware info
14855 * This function does a minimal setup of all subsystems needed for running
14858 * Returns 0 on success, negative on failure
14860 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
14862 struct i40e_vsi *vsi;
14866 pci_set_drvdata(pf->pdev, pf);
14867 pci_save_state(pf->pdev);
14869 /* set up periodic task facility */
14870 timer_setup(&pf->service_timer, i40e_service_timer, 0);
14871 pf->service_timer_period = HZ;
14873 INIT_WORK(&pf->service_task, i40e_service_task);
14874 clear_bit(__I40E_SERVICE_SCHED, pf->state);
14876 err = i40e_init_interrupt_scheme(pf);
14878 goto err_switch_setup;
14880 /* The number of VSIs reported by the FW is the minimum guaranteed
14881 * to us; HW supports far more and we share the remaining pool with
14882 * the other PFs. We allocate space for more than the guarantee with
14883 * the understanding that we might not get them all later.
14885 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
14886 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
14888 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
14890 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
14891 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
14895 goto err_switch_setup;
14898 /* We allocate one VSI which is needed as absolute minimum
14899 * in order to register the netdev
14901 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
14904 goto err_switch_setup;
14906 pf->lan_vsi = v_idx;
14907 vsi = pf->vsi[v_idx];
14910 goto err_switch_setup;
14912 vsi->alloc_queue_pairs = 1;
14913 err = i40e_config_netdev(vsi);
14915 goto err_switch_setup;
14916 err = register_netdev(vsi->netdev);
14918 goto err_switch_setup;
14919 vsi->netdev_registered = true;
14920 i40e_dbg_pf_init(pf);
14922 err = i40e_setup_misc_vector_for_recovery_mode(pf);
14924 goto err_switch_setup;
14926 /* tell the firmware that we're starting */
14927 i40e_send_version(pf);
14929 /* since everything's happy, start the service_task timer */
14930 mod_timer(&pf->service_timer,
14931 round_jiffies(jiffies + pf->service_timer_period));
14936 i40e_reset_interrupt_capability(pf);
14937 del_timer_sync(&pf->service_timer);
14938 i40e_shutdown_adminq(hw);
14939 iounmap(hw->hw_addr);
14940 pci_disable_pcie_error_reporting(pf->pdev);
14941 pci_release_mem_regions(pf->pdev);
14942 pci_disable_device(pf->pdev);
14949 * i40e_probe - Device initialization routine
14950 * @pdev: PCI device information struct
14951 * @ent: entry in i40e_pci_tbl
14953 * i40e_probe initializes a PF identified by a pci_dev structure.
14954 * The OS initialization, configuring of the PF private structure,
14955 * and a hardware reset occur.
14957 * Returns 0 on success, negative on failure
14959 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
14961 struct i40e_aq_get_phy_abilities_resp abilities;
14962 struct i40e_pf *pf;
14963 struct i40e_hw *hw;
14964 static u16 pfs_found;
14971 err = pci_enable_device_mem(pdev);
14975 /* set up for high or low dma */
14976 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
14978 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
14980 dev_err(&pdev->dev,
14981 "DMA configuration failed: 0x%x\n", err);
14986 /* set up pci connections */
14987 err = pci_request_mem_regions(pdev, i40e_driver_name);
14989 dev_info(&pdev->dev,
14990 "pci_request_selected_regions failed %d\n", err);
14994 pci_enable_pcie_error_reporting(pdev);
14995 pci_set_master(pdev);
14997 /* Now that we have a PCI connection, we need to do the
14998 * low level device setup. This is primarily setting up
14999 * the Admin Queue structures and then querying for the
15000 * device's current profile information.
15002 pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15009 set_bit(__I40E_DOWN, pf->state);
15014 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15015 I40E_MAX_CSR_SPACE);
15016 /* We believe that the highest register to read is
15017 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15018 * is not less than that before mapping to prevent a
15021 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15022 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15027 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15028 if (!hw->hw_addr) {
15030 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15031 (unsigned int)pci_resource_start(pdev, 0),
15032 pf->ioremap_len, err);
15035 hw->vendor_id = pdev->vendor;
15036 hw->device_id = pdev->device;
15037 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15038 hw->subsystem_vendor_id = pdev->subsystem_vendor;
15039 hw->subsystem_device_id = pdev->subsystem_device;
15040 hw->bus.device = PCI_SLOT(pdev->devfn);
15041 hw->bus.func = PCI_FUNC(pdev->devfn);
15042 hw->bus.bus_id = pdev->bus->number;
15043 pf->instance = pfs_found;
15045 /* Select something other than the 802.1ad ethertype for the
15046 * switch to use internally and drop on ingress.
15048 hw->switch_tag = 0xffff;
15049 hw->first_tag = ETH_P_8021AD;
15050 hw->second_tag = ETH_P_8021Q;
15052 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15053 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15054 INIT_LIST_HEAD(&pf->ddp_old_prof);
15056 /* set up the locks for the AQ, do this only once in probe
15057 * and destroy them only once in remove
15059 mutex_init(&hw->aq.asq_mutex);
15060 mutex_init(&hw->aq.arq_mutex);
15062 pf->msg_enable = netif_msg_init(debug,
15067 pf->hw.debug_mask = debug;
15069 /* do a special CORER for clearing PXE mode once at init */
15070 if (hw->revision_id == 0 &&
15071 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15072 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15077 i40e_clear_pxe_mode(hw);
15080 /* Reset here to make sure all is clean and to define PF 'n' */
15083 err = i40e_set_mac_type(hw);
15085 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15090 err = i40e_handle_resets(pf);
15094 i40e_check_recovery_mode(pf);
15096 hw->aq.num_arq_entries = I40E_AQ_LEN;
15097 hw->aq.num_asq_entries = I40E_AQ_LEN;
15098 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15099 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15100 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15102 snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15104 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15106 err = i40e_init_shared_code(hw);
15108 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15113 /* set up a default setting for link flow control */
15114 pf->hw.fc.requested_mode = I40E_FC_NONE;
15116 err = i40e_init_adminq(hw);
15118 if (err == I40E_ERR_FIRMWARE_API_VERSION)
15119 dev_info(&pdev->dev,
15120 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15121 hw->aq.api_maj_ver,
15122 hw->aq.api_min_ver,
15123 I40E_FW_API_VERSION_MAJOR,
15124 I40E_FW_MINOR_VERSION(hw));
15126 dev_info(&pdev->dev,
15127 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15131 i40e_get_oem_version(hw);
15133 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15134 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15135 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15136 hw->aq.api_maj_ver, hw->aq.api_min_ver,
15137 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15138 hw->subsystem_vendor_id, hw->subsystem_device_id);
15140 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15141 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15142 dev_dbg(&pdev->dev,
15143 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15144 hw->aq.api_maj_ver,
15145 hw->aq.api_min_ver,
15146 I40E_FW_API_VERSION_MAJOR,
15147 I40E_FW_MINOR_VERSION(hw));
15148 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15149 dev_info(&pdev->dev,
15150 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15151 hw->aq.api_maj_ver,
15152 hw->aq.api_min_ver,
15153 I40E_FW_API_VERSION_MAJOR,
15154 I40E_FW_MINOR_VERSION(hw));
15156 i40e_verify_eeprom(pf);
15158 /* Rev 0 hardware was never productized */
15159 if (hw->revision_id < 1)
15160 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15162 i40e_clear_pxe_mode(hw);
15164 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15166 goto err_adminq_setup;
15168 err = i40e_sw_init(pf);
15170 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15174 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15175 return i40e_init_recovery_mode(pf, hw);
15177 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15178 hw->func_caps.num_rx_qp, 0, 0);
15180 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15181 goto err_init_lan_hmc;
15184 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15186 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15188 goto err_configure_lan_hmc;
15191 /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15192 * Ignore error return codes because if it was already disabled via
15193 * hardware settings this will fail
15195 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15196 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15197 i40e_aq_stop_lldp(hw, true, false, NULL);
15200 /* allow a platform config to override the HW addr */
15201 i40e_get_platform_mac_addr(pdev, pf);
15203 if (!is_valid_ether_addr(hw->mac.addr)) {
15204 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15208 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15209 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15210 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15211 if (is_valid_ether_addr(hw->mac.port_addr))
15212 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15214 pci_set_drvdata(pdev, pf);
15215 pci_save_state(pdev);
15217 dev_info(&pdev->dev,
15218 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15219 "FW LLDP is disabled\n" :
15220 "FW LLDP is enabled\n");
15222 /* Enable FW to write default DCB config on link-up */
15223 i40e_aq_set_dcb_parameters(hw, true, NULL);
15225 #ifdef CONFIG_I40E_DCB
15226 err = i40e_init_pf_dcb(pf);
15228 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15229 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15230 /* Continue without DCB enabled */
15232 #endif /* CONFIG_I40E_DCB */
15234 /* set up periodic task facility */
15235 timer_setup(&pf->service_timer, i40e_service_timer, 0);
15236 pf->service_timer_period = HZ;
15238 INIT_WORK(&pf->service_task, i40e_service_task);
15239 clear_bit(__I40E_SERVICE_SCHED, pf->state);
15241 /* NVM bit on means WoL disabled for the port */
15242 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15243 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15244 pf->wol_en = false;
15247 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15249 /* set up the main switch operations */
15250 i40e_determine_queue_usage(pf);
15251 err = i40e_init_interrupt_scheme(pf);
15253 goto err_switch_setup;
15255 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15256 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15257 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15258 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15259 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15260 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15261 UDP_TUNNEL_TYPE_GENEVE;
15263 /* The number of VSIs reported by the FW is the minimum guaranteed
15264 * to us; HW supports far more and we share the remaining pool with
15265 * the other PFs. We allocate space for more than the guarantee with
15266 * the understanding that we might not get them all later.
15268 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15269 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15271 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15272 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15273 dev_warn(&pf->pdev->dev,
15274 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15275 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15276 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15279 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15280 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15284 goto err_switch_setup;
15287 #ifdef CONFIG_PCI_IOV
15288 /* prep for VF support */
15289 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15290 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15291 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15292 if (pci_num_vf(pdev))
15293 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15296 err = i40e_setup_pf_switch(pf, false, false);
15298 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15301 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15303 /* if FDIR VSI was set up, start it now */
15304 for (i = 0; i < pf->num_alloc_vsi; i++) {
15305 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15306 i40e_vsi_open(pf->vsi[i]);
15311 /* The driver only wants link up/down and module qualification
15312 * reports from firmware. Note the negative logic.
15314 err = i40e_aq_set_phy_int_mask(&pf->hw,
15315 ~(I40E_AQ_EVENT_LINK_UPDOWN |
15316 I40E_AQ_EVENT_MEDIA_NA |
15317 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15319 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15320 i40e_stat_str(&pf->hw, err),
15321 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15323 /* Reconfigure hardware for allowing smaller MSS in the case
15324 * of TSO, so that we avoid the MDD being fired and causing
15325 * a reset in the case of small MSS+TSO.
15327 val = rd32(hw, I40E_REG_MSS);
15328 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15329 val &= ~I40E_REG_MSS_MIN_MASK;
15330 val |= I40E_64BYTE_MSS;
15331 wr32(hw, I40E_REG_MSS, val);
15334 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15336 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15338 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15339 i40e_stat_str(&pf->hw, err),
15340 i40e_aq_str(&pf->hw,
15341 pf->hw.aq.asq_last_status));
15343 /* The main driver is (mostly) up and happy. We need to set this state
15344 * before setting up the misc vector or we get a race and the vector
15345 * ends up disabled forever.
15347 clear_bit(__I40E_DOWN, pf->state);
15349 /* In case of MSIX we are going to setup the misc vector right here
15350 * to handle admin queue events etc. In case of legacy and MSI
15351 * the misc functionality and queue processing is combined in
15352 * the same vector and that gets setup at open.
15354 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15355 err = i40e_setup_misc_vector(pf);
15357 dev_info(&pdev->dev,
15358 "setup of misc vector failed: %d\n", err);
15359 i40e_cloud_filter_exit(pf);
15360 i40e_fdir_teardown(pf);
15365 #ifdef CONFIG_PCI_IOV
15366 /* prep for VF support */
15367 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15368 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15369 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15370 /* disable link interrupts for VFs */
15371 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15372 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15373 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15376 if (pci_num_vf(pdev)) {
15377 dev_info(&pdev->dev,
15378 "Active VFs found, allocating resources.\n");
15379 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15381 dev_info(&pdev->dev,
15382 "Error %d allocating resources for existing VFs\n",
15386 #endif /* CONFIG_PCI_IOV */
15388 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15389 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15390 pf->num_iwarp_msix,
15391 I40E_IWARP_IRQ_PILE_ID);
15392 if (pf->iwarp_base_vector < 0) {
15393 dev_info(&pdev->dev,
15394 "failed to get tracking for %d vectors for IWARP err=%d\n",
15395 pf->num_iwarp_msix, pf->iwarp_base_vector);
15396 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15400 i40e_dbg_pf_init(pf);
15402 /* tell the firmware that we're starting */
15403 i40e_send_version(pf);
15405 /* since everything's happy, start the service_task timer */
15406 mod_timer(&pf->service_timer,
15407 round_jiffies(jiffies + pf->service_timer_period));
15409 /* add this PF to client device list and launch a client service task */
15410 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15411 err = i40e_lan_add_device(pf);
15413 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15417 #define PCI_SPEED_SIZE 8
15418 #define PCI_WIDTH_SIZE 8
15419 /* Devices on the IOSF bus do not have this information
15420 * and will report PCI Gen 1 x 1 by default so don't bother
15423 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15424 char speed[PCI_SPEED_SIZE] = "Unknown";
15425 char width[PCI_WIDTH_SIZE] = "Unknown";
15427 /* Get the negotiated link width and speed from PCI config
15430 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15433 i40e_set_pci_config_data(hw, link_status);
15435 switch (hw->bus.speed) {
15436 case i40e_bus_speed_8000:
15437 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15438 case i40e_bus_speed_5000:
15439 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15440 case i40e_bus_speed_2500:
15441 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15445 switch (hw->bus.width) {
15446 case i40e_bus_width_pcie_x8:
15447 strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15448 case i40e_bus_width_pcie_x4:
15449 strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15450 case i40e_bus_width_pcie_x2:
15451 strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15452 case i40e_bus_width_pcie_x1:
15453 strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15458 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15461 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15462 hw->bus.speed < i40e_bus_speed_8000) {
15463 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15464 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15468 /* get the requested speeds from the fw */
15469 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15471 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n",
15472 i40e_stat_str(&pf->hw, err),
15473 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15474 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15476 /* set the FEC config due to the board capabilities */
15477 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15479 /* get the supported phy types from the fw */
15480 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15482 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n",
15483 i40e_stat_str(&pf->hw, err),
15484 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15486 /* make sure the MFS hasn't been set lower than the default */
15487 #define MAX_FRAME_SIZE_DEFAULT 0x2600
15488 val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
15489 I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
15490 if (val < MAX_FRAME_SIZE_DEFAULT)
15491 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
15494 /* Add a filter to drop all Flow control frames from any VSI from being
15495 * transmitted. By doing so we stop a malicious VF from sending out
15496 * PAUSE or PFC frames and potentially controlling traffic for other
15498 * The FW can still send Flow control frames if enabled.
15500 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15501 pf->main_vsi_seid);
15503 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15504 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15505 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15506 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15507 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15508 /* print a string summarizing features */
15509 i40e_print_features(pf);
15513 /* Unwind what we've done if something failed in the setup */
15515 set_bit(__I40E_DOWN, pf->state);
15516 i40e_clear_interrupt_scheme(pf);
15519 i40e_reset_interrupt_capability(pf);
15520 del_timer_sync(&pf->service_timer);
15522 err_configure_lan_hmc:
15523 (void)i40e_shutdown_lan_hmc(hw);
15525 kfree(pf->qp_pile);
15529 iounmap(hw->hw_addr);
15533 pci_disable_pcie_error_reporting(pdev);
15534 pci_release_mem_regions(pdev);
15537 pci_disable_device(pdev);
15542 * i40e_remove - Device removal routine
15543 * @pdev: PCI device information struct
15545 * i40e_remove is called by the PCI subsystem to alert the driver
15546 * that is should release a PCI device. This could be caused by a
15547 * Hot-Plug event, or because the driver is going to be removed from
15550 static void i40e_remove(struct pci_dev *pdev)
15552 struct i40e_pf *pf = pci_get_drvdata(pdev);
15553 struct i40e_hw *hw = &pf->hw;
15554 i40e_status ret_code;
15557 i40e_dbg_pf_exit(pf);
15561 /* Disable RSS in hw */
15562 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15563 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15565 while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
15566 usleep_range(1000, 2000);
15568 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15569 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
15571 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15573 /* no more scheduling of any task */
15574 set_bit(__I40E_SUSPENDED, pf->state);
15575 set_bit(__I40E_DOWN, pf->state);
15576 if (pf->service_timer.function)
15577 del_timer_sync(&pf->service_timer);
15578 if (pf->service_task.func)
15579 cancel_work_sync(&pf->service_task);
15581 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15582 struct i40e_vsi *vsi = pf->vsi[0];
15584 /* We know that we have allocated only one vsi for this PF,
15585 * it was just for registering netdevice, so the interface
15586 * could be visible in the 'ifconfig' output
15588 unregister_netdev(vsi->netdev);
15589 free_netdev(vsi->netdev);
15594 /* Client close must be called explicitly here because the timer
15595 * has been stopped.
15597 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15599 i40e_fdir_teardown(pf);
15601 /* If there is a switch structure or any orphans, remove them.
15602 * This will leave only the PF's VSI remaining.
15604 for (i = 0; i < I40E_MAX_VEB; i++) {
15608 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15609 pf->veb[i]->uplink_seid == 0)
15610 i40e_switch_branch_release(pf->veb[i]);
15613 /* Now we can shutdown the PF's VSIs, just before we kill
15616 for (i = pf->num_alloc_vsi; i--;)
15618 i40e_vsi_close(pf->vsi[i]);
15619 i40e_vsi_release(pf->vsi[i]);
15623 i40e_cloud_filter_exit(pf);
15625 /* remove attached clients */
15626 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15627 ret_code = i40e_lan_del_device(pf);
15629 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15633 /* shutdown and destroy the HMC */
15634 if (hw->hmc.hmc_obj) {
15635 ret_code = i40e_shutdown_lan_hmc(hw);
15637 dev_warn(&pdev->dev,
15638 "Failed to destroy the HMC resources: %d\n",
15643 /* Free MSI/legacy interrupt 0 when in recovery mode. */
15644 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15645 !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15646 free_irq(pf->pdev->irq, pf);
15648 /* shutdown the adminq */
15649 i40e_shutdown_adminq(hw);
15651 /* destroy the locks only once, here */
15652 mutex_destroy(&hw->aq.arq_mutex);
15653 mutex_destroy(&hw->aq.asq_mutex);
15655 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
15657 i40e_clear_interrupt_scheme(pf);
15658 for (i = 0; i < pf->num_alloc_vsi; i++) {
15660 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
15661 i40e_vsi_clear_rings(pf->vsi[i]);
15662 i40e_vsi_clear(pf->vsi[i]);
15668 for (i = 0; i < I40E_MAX_VEB; i++) {
15673 kfree(pf->qp_pile);
15676 iounmap(hw->hw_addr);
15678 pci_release_mem_regions(pdev);
15680 pci_disable_pcie_error_reporting(pdev);
15681 pci_disable_device(pdev);
15685 * i40e_pci_error_detected - warning that something funky happened in PCI land
15686 * @pdev: PCI device information struct
15687 * @error: the type of PCI error
15689 * Called to warn that something happened and the error handling steps
15690 * are in progress. Allows the driver to quiesce things, be ready for
15693 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
15694 pci_channel_state_t error)
15696 struct i40e_pf *pf = pci_get_drvdata(pdev);
15698 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
15701 dev_info(&pdev->dev,
15702 "Cannot recover - error happened during device probe\n");
15703 return PCI_ERS_RESULT_DISCONNECT;
15706 /* shutdown all operations */
15707 if (!test_bit(__I40E_SUSPENDED, pf->state))
15708 i40e_prep_for_reset(pf, false);
15710 /* Request a slot reset */
15711 return PCI_ERS_RESULT_NEED_RESET;
15715 * i40e_pci_error_slot_reset - a PCI slot reset just happened
15716 * @pdev: PCI device information struct
15718 * Called to find if the driver can work with the device now that
15719 * the pci slot has been reset. If a basic connection seems good
15720 * (registers are readable and have sane content) then return a
15721 * happy little PCI_ERS_RESULT_xxx.
15723 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
15725 struct i40e_pf *pf = pci_get_drvdata(pdev);
15726 pci_ers_result_t result;
15729 dev_dbg(&pdev->dev, "%s\n", __func__);
15730 if (pci_enable_device_mem(pdev)) {
15731 dev_info(&pdev->dev,
15732 "Cannot re-enable PCI device after reset.\n");
15733 result = PCI_ERS_RESULT_DISCONNECT;
15735 pci_set_master(pdev);
15736 pci_restore_state(pdev);
15737 pci_save_state(pdev);
15738 pci_wake_from_d3(pdev, false);
15740 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
15742 result = PCI_ERS_RESULT_RECOVERED;
15744 result = PCI_ERS_RESULT_DISCONNECT;
15751 * i40e_pci_error_reset_prepare - prepare device driver for pci reset
15752 * @pdev: PCI device information struct
15754 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
15756 struct i40e_pf *pf = pci_get_drvdata(pdev);
15758 i40e_prep_for_reset(pf, false);
15762 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
15763 * @pdev: PCI device information struct
15765 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
15767 struct i40e_pf *pf = pci_get_drvdata(pdev);
15769 i40e_reset_and_rebuild(pf, false, false);
15770 #ifdef CONFIG_PCI_IOV
15771 i40e_restore_all_vfs_msi_state(pdev);
15772 #endif /* CONFIG_PCI_IOV */
15776 * i40e_pci_error_resume - restart operations after PCI error recovery
15777 * @pdev: PCI device information struct
15779 * Called to allow the driver to bring things back up after PCI error
15780 * and/or reset recovery has finished.
15782 static void i40e_pci_error_resume(struct pci_dev *pdev)
15784 struct i40e_pf *pf = pci_get_drvdata(pdev);
15786 dev_dbg(&pdev->dev, "%s\n", __func__);
15787 if (test_bit(__I40E_SUSPENDED, pf->state))
15790 i40e_handle_reset_warning(pf, false);
15794 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
15795 * using the mac_address_write admin q function
15796 * @pf: pointer to i40e_pf struct
15798 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
15800 struct i40e_hw *hw = &pf->hw;
15805 /* Get current MAC address in case it's an LAA */
15806 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
15807 ether_addr_copy(mac_addr,
15808 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
15810 dev_err(&pf->pdev->dev,
15811 "Failed to retrieve MAC address; using default\n");
15812 ether_addr_copy(mac_addr, hw->mac.addr);
15815 /* The FW expects the mac address write cmd to first be called with
15816 * one of these flags before calling it again with the multicast
15819 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
15821 if (hw->func_caps.flex10_enable && hw->partition_id != 1)
15822 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
15824 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15826 dev_err(&pf->pdev->dev,
15827 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
15831 flags = I40E_AQC_MC_MAG_EN
15832 | I40E_AQC_WOL_PRESERVE_ON_PFR
15833 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
15834 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15836 dev_err(&pf->pdev->dev,
15837 "Failed to enable Multicast Magic Packet wake up\n");
15841 * i40e_shutdown - PCI callback for shutting down
15842 * @pdev: PCI device information struct
15844 static void i40e_shutdown(struct pci_dev *pdev)
15846 struct i40e_pf *pf = pci_get_drvdata(pdev);
15847 struct i40e_hw *hw = &pf->hw;
15849 set_bit(__I40E_SUSPENDED, pf->state);
15850 set_bit(__I40E_DOWN, pf->state);
15852 del_timer_sync(&pf->service_timer);
15853 cancel_work_sync(&pf->service_task);
15854 i40e_cloud_filter_exit(pf);
15855 i40e_fdir_teardown(pf);
15857 /* Client close must be called explicitly here because the timer
15858 * has been stopped.
15860 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15862 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15863 i40e_enable_mc_magic_wake(pf);
15865 i40e_prep_for_reset(pf, false);
15867 wr32(hw, I40E_PFPM_APM,
15868 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15869 wr32(hw, I40E_PFPM_WUFC,
15870 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15872 /* Free MSI/legacy interrupt 0 when in recovery mode. */
15873 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15874 !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15875 free_irq(pf->pdev->irq, pf);
15877 /* Since we're going to destroy queues during the
15878 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
15882 i40e_clear_interrupt_scheme(pf);
15885 if (system_state == SYSTEM_POWER_OFF) {
15886 pci_wake_from_d3(pdev, pf->wol_en);
15887 pci_set_power_state(pdev, PCI_D3hot);
15892 * i40e_suspend - PM callback for moving to D3
15893 * @dev: generic device information structure
15895 static int __maybe_unused i40e_suspend(struct device *dev)
15897 struct i40e_pf *pf = dev_get_drvdata(dev);
15898 struct i40e_hw *hw = &pf->hw;
15900 /* If we're already suspended, then there is nothing to do */
15901 if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
15904 set_bit(__I40E_DOWN, pf->state);
15906 /* Ensure service task will not be running */
15907 del_timer_sync(&pf->service_timer);
15908 cancel_work_sync(&pf->service_task);
15910 /* Client close must be called explicitly here because the timer
15911 * has been stopped.
15913 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15915 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15916 i40e_enable_mc_magic_wake(pf);
15918 /* Since we're going to destroy queues during the
15919 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
15924 i40e_prep_for_reset(pf, true);
15926 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15927 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15929 /* Clear the interrupt scheme and release our IRQs so that the system
15930 * can safely hibernate even when there are a large number of CPUs.
15931 * Otherwise hibernation might fail when mapping all the vectors back
15934 i40e_clear_interrupt_scheme(pf);
15942 * i40e_resume - PM callback for waking up from D3
15943 * @dev: generic device information structure
15945 static int __maybe_unused i40e_resume(struct device *dev)
15947 struct i40e_pf *pf = dev_get_drvdata(dev);
15950 /* If we're not suspended, then there is nothing to do */
15951 if (!test_bit(__I40E_SUSPENDED, pf->state))
15954 /* We need to hold the RTNL lock prior to restoring interrupt schemes,
15955 * since we're going to be restoring queues
15959 /* We cleared the interrupt scheme when we suspended, so we need to
15960 * restore it now to resume device functionality.
15962 err = i40e_restore_interrupt_scheme(pf);
15964 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
15968 clear_bit(__I40E_DOWN, pf->state);
15969 i40e_reset_and_rebuild(pf, false, true);
15973 /* Clear suspended state last after everything is recovered */
15974 clear_bit(__I40E_SUSPENDED, pf->state);
15976 /* Restart the service task */
15977 mod_timer(&pf->service_timer,
15978 round_jiffies(jiffies + pf->service_timer_period));
15983 static const struct pci_error_handlers i40e_err_handler = {
15984 .error_detected = i40e_pci_error_detected,
15985 .slot_reset = i40e_pci_error_slot_reset,
15986 .reset_prepare = i40e_pci_error_reset_prepare,
15987 .reset_done = i40e_pci_error_reset_done,
15988 .resume = i40e_pci_error_resume,
15991 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
15993 static struct pci_driver i40e_driver = {
15994 .name = i40e_driver_name,
15995 .id_table = i40e_pci_tbl,
15996 .probe = i40e_probe,
15997 .remove = i40e_remove,
15999 .pm = &i40e_pm_ops,
16001 .shutdown = i40e_shutdown,
16002 .err_handler = &i40e_err_handler,
16003 .sriov_configure = i40e_pci_sriov_configure,
16007 * i40e_init_module - Driver registration routine
16009 * i40e_init_module is the first routine called when the driver is
16010 * loaded. All it does is register with the PCI subsystem.
16012 static int __init i40e_init_module(void)
16016 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16017 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16019 /* There is no need to throttle the number of active tasks because
16020 * each device limits its own task using a state bit for scheduling
16021 * the service task, and the device tasks do not interfere with each
16022 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16023 * since we need to be able to guarantee forward progress even under
16026 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16028 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16033 err = pci_register_driver(&i40e_driver);
16035 destroy_workqueue(i40e_wq);
16042 module_init(i40e_init_module);
16045 * i40e_exit_module - Driver exit cleanup routine
16047 * i40e_exit_module is called just before the driver is removed
16050 static void __exit i40e_exit_module(void)
16052 pci_unregister_driver(&i40e_driver);
16053 destroy_workqueue(i40e_wq);
16056 module_exit(i40e_exit_module);