1 /* QLogic qede NIC Driver
2 * Copyright (c) 2015-2017 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/compiler.h>
35 #include <linux/version.h>
36 #include <linux/workqueue.h>
37 #include <linux/netdevice.h>
38 #include <linux/interrupt.h>
39 #include <linux/bitmap.h>
40 #include <linux/kernel.h>
41 #include <linux/mutex.h>
42 #include <linux/bpf.h>
44 #include <linux/qed/qede_rdma.h>
46 #ifdef CONFIG_RFS_ACCEL
47 #include <linux/cpu_rmap.h>
49 #include <linux/qed/common_hsi.h>
50 #include <linux/qed/eth_common.h>
51 #include <linux/qed/qed_if.h>
52 #include <linux/qed/qed_chain.h>
53 #include <linux/qed/qed_eth_if.h>
55 #include <net/pkt_cls.h>
56 #include <net/tc_act/tc_gact.h>
58 #define QEDE_MAJOR_VERSION 8
59 #define QEDE_MINOR_VERSION 33
60 #define QEDE_REVISION_VERSION 0
61 #define QEDE_ENGINEERING_VERSION 20
62 #define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "." \
63 __stringify(QEDE_MINOR_VERSION) "." \
64 __stringify(QEDE_REVISION_VERSION) "." \
65 __stringify(QEDE_ENGINEERING_VERSION)
67 #define DRV_MODULE_SYM qede
69 struct qede_stats_common {
71 u64 packet_too_big_discard;
79 u64 mftag_filter_discards;
80 u64 mac_filter_discards;
91 u64 coalesced_aborts_num;
92 u64 non_coalesced_pkts;
94 u64 link_change_count;
97 u64 rx_64_byte_packets;
98 u64 rx_65_to_127_byte_packets;
99 u64 rx_128_to_255_byte_packets;
100 u64 rx_256_to_511_byte_packets;
101 u64 rx_512_to_1023_byte_packets;
102 u64 rx_1024_to_1518_byte_packets;
104 u64 rx_mac_crtl_frames;
108 u64 rx_carrier_errors;
109 u64 rx_oversize_packets;
111 u64 rx_undersize_packets;
113 u64 tx_64_byte_packets;
114 u64 tx_65_to_127_byte_packets;
115 u64 tx_128_to_255_byte_packets;
116 u64 tx_256_to_511_byte_packets;
117 u64 tx_512_to_1023_byte_packets;
118 u64 tx_1024_to_1518_byte_packets;
123 u64 tx_mac_ctrl_frames;
126 struct qede_stats_bb {
127 u64 rx_1519_to_1522_byte_packets;
128 u64 rx_1519_to_2047_byte_packets;
129 u64 rx_2048_to_4095_byte_packets;
130 u64 rx_4096_to_9216_byte_packets;
131 u64 rx_9217_to_16383_byte_packets;
132 u64 tx_1519_to_2047_byte_packets;
133 u64 tx_2048_to_4095_byte_packets;
134 u64 tx_4096_to_9216_byte_packets;
135 u64 tx_9217_to_16383_byte_packets;
136 u64 tx_lpi_entry_count;
137 u64 tx_total_collisions;
140 struct qede_stats_ah {
141 u64 rx_1519_to_max_byte_packets;
142 u64 tx_1519_to_max_byte_packets;
146 struct qede_stats_common common;
149 struct qede_stats_bb bb;
150 struct qede_stats_ah ah;
155 struct list_head list;
160 struct qede_rdma_dev {
161 struct qedr_dev *qedr_dev;
162 struct list_head entry;
163 struct list_head rdma_event_list;
164 struct workqueue_struct *rdma_wq;
166 struct completion event_comp;
171 #define QEDE_RFS_MAX_FLTR 256
174 struct qed_dev *cdev;
175 struct net_device *ndev;
176 struct pci_dev *pdev;
182 #define QEDE_FLAG_IS_VF BIT(0)
183 #define IS_VF(edev) (!!((edev)->flags & QEDE_FLAG_IS_VF))
184 #define QEDE_TX_TIMESTAMPING_EN BIT(1)
185 #define QEDE_FLAGS_PTP_TX_IN_PRORGESS BIT(2)
187 const struct qed_eth_ops *ops;
188 struct qede_ptp *ptp;
190 struct qed_dev_eth_info dev_info;
191 #define QEDE_MAX_RSS_CNT(edev) ((edev)->dev_info.num_queues)
192 #define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues)
193 #define QEDE_IS_BB(edev) \
194 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_BB)
195 #define QEDE_IS_AH(edev) \
196 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_AH)
198 struct qede_fastpath *fp_array;
205 #define QEDE_QUEUE_CNT(edev) ((edev)->num_queues)
206 #define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx)
207 #define QEDE_RX_QUEUE_IDX(edev, i) (i)
208 #define QEDE_TSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_rx)
210 struct qed_int_info int_info;
212 /* Smaller private varaiant of the RTNL lock */
213 struct mutex qede_lock;
214 u32 state; /* Protected by qede_lock */
218 /* L2 header size + 2*VLANs (8 bytes) + LLC SNAP (8 bytes) */
219 #define ETH_OVERHEAD (ETH_HLEN + 8 + 8)
220 /* Max supported alignment is 256 (8 shift)
221 * minimal alignment shift 6 is optimal for 57xxx HW performance
223 #define QEDE_RX_ALIGN_SHIFT max(6, min(8, L1_CACHE_SHIFT))
224 /* We assume skb_build() uses sizeof(struct skb_shared_info) bytes
225 * at the end of skb->data, to avoid wasting a full cache line.
226 * This reduces memory use (skb->truesize).
228 #define QEDE_FW_RX_ALIGN_END \
229 max_t(u64, 1UL << QEDE_RX_ALIGN_SHIFT, \
230 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
232 struct qede_stats stats;
233 #define QEDE_RSS_INDIR_INITED BIT(0)
234 #define QEDE_RSS_KEY_INITED BIT(1)
235 #define QEDE_RSS_CAPS_INITED BIT(2)
236 u32 rss_params_inited; /* bit-field to track initialized rss params */
237 u16 rss_ind_table[128];
241 u16 q_num_rx_buffers; /* Must be a power of two */
242 u16 q_num_tx_buffers; /* Must be a power of two */
245 struct list_head vlan_list;
246 u16 configured_vlans;
247 u16 non_configured_vlans;
248 bool accept_any_vlan;
249 struct delayed_work sp_task;
250 unsigned long sp_flags;
254 struct qede_arfs *arfs;
257 struct qede_rdma_dev rdma_info;
259 struct bpf_prog *xdp_prog;
267 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
270 #define MAX_NUM_PRI 8
272 /* The driver supports the new build_skb() API:
273 * RX ring buffer contains pointer to kmalloc() data only,
274 * skb are built only after the frame was DMA-ed.
279 unsigned int page_offset;
282 enum qede_agg_state {
283 QEDE_AGG_STATE_NONE = 0,
284 QEDE_AGG_STATE_START = 1,
285 QEDE_AGG_STATE_ERROR = 2
288 struct qede_agg_info {
289 /* rx_buf is a data buffer that can be placed / consumed from rx bd
290 * chain. It has two purposes: We will preallocate the data buffer
291 * for each aggregation when we open the interface and will place this
292 * buffer on the rx-bd-ring when we receive TPA_START. We don't want
293 * to be in a state where allocation fails, as we can't reuse the
294 * consumer buffer in the rx-chain since FW may still be writing to it
295 * (since header needs to be modified for TPA).
296 * The second purpose is to keep a pointer to the bd buffer during
299 struct sw_rx_data buffer;
302 /* We need some structs from the start cookie until termination */
312 struct qede_rx_queue {
314 void __iomem *hw_rxq_prod_addr;
316 /* Required for the allocation of replacement buffers */
319 struct bpf_prog *xdp_prog;
328 /* Used once per each NAPI run */
336 struct sw_rx_data *sw_rx_ring;
337 struct qed_chain rx_bd_ring;
338 struct qed_chain rx_comp_ring ____cacheline_aligned;
341 struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM];
343 /* Used once per each NAPI run */
353 struct xdp_rxq_info xdp_rxq;
357 struct eth_db_data data;
364 /* Set on the first BD descriptor when there is a split BD */
365 #define QEDE_TSO_SPLIT_BD BIT(0)
373 struct qede_tx_queue {
378 u16 num_tx_buffers; /* Slowpath only */
385 /* Needed for the mapping of packets */
388 void __iomem *doorbell_addr;
390 int index; /* Slowpath only */
391 #define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \
392 QEDE_MAX_TSS_CNT(edev))
393 #define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev))
394 #define QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx) ((edev)->fp_num_rx + \
395 ((idx) % QEDE_TSS_COUNT(edev)))
396 #define QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx) ((idx) / QEDE_TSS_COUNT(edev))
397 #define QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq) ((QEDE_TSS_COUNT(edev) * \
398 (txq)->cos) + (txq)->index)
399 #define QEDE_NDEV_TXQ_ID_TO_TXQ(edev, idx) \
400 (&((edev)->fp_array[QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx)].txq \
401 [QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx)]))
402 #define QEDE_FP_TC0_TXQ(fp) (&((fp)->txq[0]))
404 /* Regular Tx requires skb + metadata for release purpose,
405 * while XDP requires the pages and the mapped address.
408 struct sw_tx_bd *skbs;
409 struct sw_tx_xdp *xdp;
412 struct qed_chain tx_pbl;
414 /* Slowpath; Should be kept in end [unless missing padding] */
420 #define BD_UNMAP_ADDR(bd) HILO_U64(le32_to_cpu((bd)->addr.hi), \
421 le32_to_cpu((bd)->addr.lo))
422 #define BD_SET_UNMAP_ADDR_LEN(bd, maddr, len) \
424 (bd)->addr.hi = cpu_to_le32(upper_32_bits(maddr)); \
425 (bd)->addr.lo = cpu_to_le32(lower_32_bits(maddr)); \
426 (bd)->nbytes = cpu_to_le16(len); \
428 #define BD_UNMAP_LEN(bd) (le16_to_cpu((bd)->nbytes))
430 struct qede_fastpath {
431 struct qede_dev *edev;
432 #define QEDE_FASTPATH_TX BIT(0)
433 #define QEDE_FASTPATH_RX BIT(1)
434 #define QEDE_FASTPATH_XDP BIT(2)
435 #define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX)
439 struct napi_struct napi;
440 struct qed_sb_info *sb_info;
441 struct qede_rx_queue *rxq;
442 struct qede_tx_queue *txq;
443 struct qede_tx_queue *xdp_tx;
445 #define VEC_NAME_SIZE (sizeof(((struct net_device *)0)->name) + 8)
446 char name[VEC_NAME_SIZE];
449 /* Debug print definitions */
450 #define DP_NAME(edev) ((edev)->ndev->name)
453 #define XMIT_L4_CSUM BIT(0)
454 #define XMIT_LSO BIT(1)
455 #define XMIT_ENC BIT(2)
456 #define XMIT_ENC_GSO_L4_CSUM BIT(3)
458 #define QEDE_CSUM_ERROR BIT(0)
459 #define QEDE_CSUM_UNNECESSARY BIT(1)
460 #define QEDE_TUNN_CSUM_UNNECESSARY BIT(2)
462 #define QEDE_SP_RX_MODE 1
464 #ifdef CONFIG_RFS_ACCEL
465 int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
466 u16 rxq_index, u32 flow_id);
467 #define QEDE_SP_ARFS_CONFIG 4
468 #define QEDE_SP_TASK_POLL_DELAY (5 * HZ)
471 void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr);
472 void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev);
473 void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc);
474 void qede_free_arfs(struct qede_dev *edev);
475 int qede_alloc_arfs(struct qede_dev *edev);
476 int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info);
477 int qede_delete_flow_filter(struct qede_dev *edev, u64 cookie);
478 int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd);
479 int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info,
481 int qede_get_arfs_filter_count(struct qede_dev *edev);
483 struct qede_reload_args {
484 void (*func)(struct qede_dev *edev, struct qede_reload_args *args);
486 netdev_features_t features;
487 struct bpf_prog *new_prog;
492 /* Datapath functions definition */
493 netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev);
494 u16 qede_select_queue(struct net_device *dev, struct sk_buff *skb,
495 struct net_device *sb_dev,
496 select_queue_fallback_t fallback);
497 netdev_features_t qede_features_check(struct sk_buff *skb,
498 struct net_device *dev,
499 netdev_features_t features);
500 void qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp);
501 int qede_alloc_rx_buffer(struct qede_rx_queue *rxq, bool allow_lazy);
502 int qede_free_tx_pkt(struct qede_dev *edev,
503 struct qede_tx_queue *txq, int *len);
504 int qede_poll(struct napi_struct *napi, int budget);
505 irqreturn_t qede_msix_fp_int(int irq, void *fp_cookie);
507 /* Filtering function definitions */
508 void qede_force_mac(void *dev, u8 *mac, bool forced);
509 void qede_udp_ports_update(void *dev, u16 vxlan_port, u16 geneve_port);
510 int qede_set_mac_addr(struct net_device *ndev, void *p);
512 int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid);
513 int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid);
514 void qede_vlan_mark_nonconfigured(struct qede_dev *edev);
515 int qede_configure_vlan_filters(struct qede_dev *edev);
517 netdev_features_t qede_fix_features(struct net_device *dev,
518 netdev_features_t features);
519 int qede_set_features(struct net_device *dev, netdev_features_t features);
520 void qede_set_rx_mode(struct net_device *ndev);
521 void qede_config_rx_mode(struct net_device *ndev);
522 void qede_fill_rss_params(struct qede_dev *edev,
523 struct qed_update_vport_rss_params *rss, u8 *update);
525 void qede_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti);
526 void qede_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti);
528 int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp);
531 void qede_set_dcbnl_ops(struct net_device *ndev);
534 void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level);
535 void qede_set_ethtool_ops(struct net_device *netdev);
536 void qede_reload(struct qede_dev *edev,
537 struct qede_reload_args *args, bool is_locked);
538 int qede_change_mtu(struct net_device *dev, int new_mtu);
539 void qede_fill_by_demand_stats(struct qede_dev *edev);
540 void __qede_lock(struct qede_dev *edev);
541 void __qede_unlock(struct qede_dev *edev);
542 bool qede_has_rx_work(struct qede_rx_queue *rxq);
543 int qede_txq_has_work(struct qede_tx_queue *txq);
544 void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count);
545 void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq);
546 int qede_add_tc_flower_fltr(struct qede_dev *edev, __be16 proto,
547 struct tc_cls_flower_offload *f);
549 #define RX_RING_SIZE_POW 13
550 #define RX_RING_SIZE ((u16)BIT(RX_RING_SIZE_POW))
551 #define NUM_RX_BDS_MAX (RX_RING_SIZE - 1)
552 #define NUM_RX_BDS_MIN 128
553 #define NUM_RX_BDS_KDUMP_MIN 63
554 #define NUM_RX_BDS_DEF ((u16)BIT(10) - 1)
556 #define TX_RING_SIZE_POW 13
557 #define TX_RING_SIZE ((u16)BIT(TX_RING_SIZE_POW))
558 #define NUM_TX_BDS_MAX (TX_RING_SIZE - 1)
559 #define NUM_TX_BDS_MIN 128
560 #define NUM_TX_BDS_KDUMP_MIN 63
561 #define NUM_TX_BDS_DEF NUM_TX_BDS_MAX
563 #define QEDE_MIN_PKT_LEN 64
564 #define QEDE_RX_HDR_SIZE 256
565 #define QEDE_MAX_JUMBO_PACKET_SIZE 9600
566 #define for_each_queue(i) for (i = 0; i < edev->num_queues; i++)
567 #define for_each_cos_in_txq(edev, var) \
568 for ((var) = 0; (var) < (edev)->dev_info.num_tc; (var)++)
570 #endif /* _QEDE_H_ */