drivers/net/ethernet/hisilicon/hns3/hns3_enet.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/if_vlan.h>
   8 #include <linux/ip.h>
   9 #include <linux/ipv6.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/sctp.h>
  14 #include <linux/vermagic.h>
  15 #include <net/gre.h>
  16 #include <net/pkt_cls.h>
  17 #include <net/vxlan.h>
  18
  19 #include "hnae3.h"
  20 #include "hns3_enet.h"
  21
  22 static void hns3_clear_all_ring(struct hnae3_handle *h);
  23 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
  24
  25 static const char hns3_driver_name[] = "hns3";
  26 const char hns3_driver_version[] = VERMAGIC_STRING;
  27 static const char hns3_driver_string[] =
  28                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  29 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  30 static struct hnae3_client client;
  31
  32 #define HNS3_MIN_TUN_PKT_LEN    65U
  33
  34 /* hns3_pci_tbl - PCI Device ID Table
  35  *
  36  * Last entry must be all 0s
  37  *
  38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  39  *   Class, Class Mask, private data (not used) }
  40  */
  41 static const struct pci_device_id hns3_pci_tbl[] = {
  42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  45          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  47          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  49          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  50         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  51          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  52         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  53          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  54         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
  55         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
  56          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  57         /* required last entry */
  58         {0, }
  59 };
  60 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  61
  62 static irqreturn_t hns3_irq_handle(int irq, void *vector)
  63 {
  64         struct hns3_enet_tqp_vector *tqp_vector = vector;
  65
  66         napi_schedule(&tqp_vector->napi);
  67
  68         return IRQ_HANDLED;
  69 }
  70
  71 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
  72 {
  73         struct hns3_enet_tqp_vector *tqp_vectors;
  74         unsigned int i;
  75
  76         for (i = 0; i < priv->vector_num; i++) {
  77                 tqp_vectors = &priv->tqp_vector[i];
  78
  79                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
  80                         continue;
  81
  82                 /* release the irq resource */
  83                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
  84                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
  85         }
  86 }
  87
  88 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
  89 {
  90         struct hns3_enet_tqp_vector *tqp_vectors;
  91         int txrx_int_idx = 0;
  92         int rx_int_idx = 0;
  93         int tx_int_idx = 0;
  94         unsigned int i;
  95         int ret;
  96
  97         for (i = 0; i < priv->vector_num; i++) {
  98                 tqp_vectors = &priv->tqp_vector[i];
  99
 100                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 101                         continue;
 102
 103                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 104                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 105                                  "%s-%s-%d", priv->netdev->name, "TxRx",
 106                                  txrx_int_idx++);
 107                         txrx_int_idx++;
 108                 } else if (tqp_vectors->rx_group.ring) {
 109                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 110                                  "%s-%s-%d", priv->netdev->name, "Rx",
 111                                  rx_int_idx++);
 112                 } else if (tqp_vectors->tx_group.ring) {
 113                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 114                                  "%s-%s-%d", priv->netdev->name, "Tx",
 115                                  tx_int_idx++);
 116                 } else {
 117                         /* Skip this unused q_vector */
 118                         continue;
 119                 }
 120
 121                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 122
 123                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 124                                   tqp_vectors->name,
 125                                        tqp_vectors);
 126                 if (ret) {
 127                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 128                                    tqp_vectors->vector_irq);
 129                         return ret;
 130                 }
 131
 132                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 133         }
 134
 135         return 0;
 136 }
 137
 138 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 139                                  u32 mask_en)
 140 {
 141         writel(mask_en, tqp_vector->mask_addr);
 142 }
 143
 144 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 145 {
 146         napi_enable(&tqp_vector->napi);
 147
 148         /* enable vector */
 149         hns3_mask_vector_irq(tqp_vector, 1);
 150 }
 151
 152 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 153 {
 154         /* disable vector */
 155         hns3_mask_vector_irq(tqp_vector, 0);
 156
 157         disable_irq(tqp_vector->vector_irq);
 158         napi_disable(&tqp_vector->napi);
 159 }
 160
 161 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 162                                  u32 rl_value)
 163 {
 164         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 165
 166         /* this defines the configuration for RL (Interrupt Rate Limiter).
 167          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 168          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 169          */
 170
 171         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
 172             !tqp_vector->rx_group.coal.gl_adapt_enable)
 173                 /* According to the hardware, the range of rl_reg is
 174                  * 0-59 and the unit is 4.
 175                  */
 176                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 177
 178         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 179 }
 180
 181 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 182                                     u32 gl_value)
 183 {
 184         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 185
 186         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 187 }
 188
 189 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 190                                     u32 gl_value)
 191 {
 192         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 193
 194         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 195 }
 196
 197 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
 198                                    struct hns3_nic_priv *priv)
 199 {
 200         /* initialize the configuration for interrupt coalescing.
 201          * 1. GL (Interrupt Gap Limiter)
 202          * 2. RL (Interrupt Rate Limiter)
 203          */
 204
 205         /* Default: enable interrupt coalescing self-adaptive and GL */
 206         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
 207         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
 208
 209         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
 210         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
 211
 212         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
 213         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
 214         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
 215 }
 216
 217 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 218                                       struct hns3_nic_priv *priv)
 219 {
 220         struct hnae3_handle *h = priv->ae_handle;
 221
 222         hns3_set_vector_coalesce_tx_gl(tqp_vector,
 223                                        tqp_vector->tx_group.coal.int_gl);
 224         hns3_set_vector_coalesce_rx_gl(tqp_vector,
 225                                        tqp_vector->rx_group.coal.int_gl);
 226         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 227 }
 228
 229 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 230 {
 231         struct hnae3_handle *h = hns3_get_handle(netdev);
 232         struct hnae3_knic_private_info *kinfo = &h->kinfo;
 233         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
 234         int i, ret;
 235
 236         if (kinfo->num_tc <= 1) {
 237                 netdev_reset_tc(netdev);
 238         } else {
 239                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
 240                 if (ret) {
 241                         netdev_err(netdev,
 242                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
 243                         return ret;
 244                 }
 245
 246                 for (i = 0; i < HNAE3_MAX_TC; i++) {
 247                         if (!kinfo->tc_info[i].enable)
 248                                 continue;
 249
 250                         netdev_set_tc_queue(netdev,
 251                                             kinfo->tc_info[i].tc,
 252                                             kinfo->tc_info[i].tqp_count,
 253                                             kinfo->tc_info[i].tqp_offset);
 254                 }
 255         }
 256
 257         ret = netif_set_real_num_tx_queues(netdev, queue_size);
 258         if (ret) {
 259                 netdev_err(netdev,
 260                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
 261                            ret);
 262                 return ret;
 263         }
 264
 265         ret = netif_set_real_num_rx_queues(netdev, queue_size);
 266         if (ret) {
 267                 netdev_err(netdev,
 268                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 269                 return ret;
 270         }
 271
 272         return 0;
 273 }
 274
 275 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 276 {
 277         u16 free_tqps, max_rss_size, max_tqps;
 278
 279         h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
 280         max_tqps = h->kinfo.num_tc * max_rss_size;
 281
 282         return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
 283 }
 284
 285 static int hns3_nic_net_up(struct net_device *netdev)
 286 {
 287         struct hns3_nic_priv *priv = netdev_priv(netdev);
 288         struct hnae3_handle *h = priv->ae_handle;
 289         int i, j;
 290         int ret;
 291
 292         ret = hns3_nic_reset_all_ring(h);
 293         if (ret)
 294                 return ret;
 295
 296         /* get irq resource for all vectors */
 297         ret = hns3_nic_init_irq(priv);
 298         if (ret) {
 299                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
 300                 return ret;
 301         }
 302
 303         /* enable the vectors */
 304         for (i = 0; i < priv->vector_num; i++)
 305                 hns3_vector_enable(&priv->tqp_vector[i]);
 306
 307         /* start the ae_dev */
 308         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 309         if (ret)
 310                 goto out_start_err;
 311
 312         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 313
 314         return 0;
 315
 316 out_start_err:
 317         for (j = i - 1; j >= 0; j--)
 318                 hns3_vector_disable(&priv->tqp_vector[j]);
 319
 320         hns3_nic_uninit_irq(priv);
 321
 322         return ret;
 323 }
 324
 325 static int hns3_nic_net_open(struct net_device *netdev)
 326 {
 327         struct hns3_nic_priv *priv = netdev_priv(netdev);
 328         struct hnae3_handle *h = hns3_get_handle(netdev);
 329         struct hnae3_knic_private_info *kinfo;
 330         int i, ret;
 331
 332         netif_carrier_off(netdev);
 333
 334         ret = hns3_nic_set_real_num_queue(netdev);
 335         if (ret)
 336                 return ret;
 337
 338         ret = hns3_nic_net_up(netdev);
 339         if (ret) {
 340                 netdev_err(netdev,
 341                            "hns net up fail, ret=%d!\n", ret);
 342                 return ret;
 343         }
 344
 345         kinfo = &h->kinfo;
 346         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
 347                 netdev_set_prio_tc_map(netdev, i,
 348                                        kinfo->prio_tc[i]);
 349         }
 350
 351         priv->ae_handle->last_reset_time = jiffies;
 352         return 0;
 353 }
 354
 355 static void hns3_nic_net_down(struct net_device *netdev)
 356 {
 357         struct hns3_nic_priv *priv = netdev_priv(netdev);
 358         const struct hnae3_ae_ops *ops;
 359         int i;
 360
 361         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 362                 return;
 363
 364         /* disable vectors */
 365         for (i = 0; i < priv->vector_num; i++)
 366                 hns3_vector_disable(&priv->tqp_vector[i]);
 367
 368         /* stop ae_dev */
 369         ops = priv->ae_handle->ae_algo->ops;
 370         if (ops->stop)
 371                 ops->stop(priv->ae_handle);
 372
 373         /* free irq resources */
 374         hns3_nic_uninit_irq(priv);
 375
 376         hns3_clear_all_ring(priv->ae_handle);
 377 }
 378
 379 static int hns3_nic_net_stop(struct net_device *netdev)
 380 {
 381         netif_tx_stop_all_queues(netdev);
 382         netif_carrier_off(netdev);
 383
 384         hns3_nic_net_down(netdev);
 385
 386         return 0;
 387 }
 388
 389 static int hns3_nic_uc_sync(struct net_device *netdev,
 390                             const unsigned char *addr)
 391 {
 392         struct hnae3_handle *h = hns3_get_handle(netdev);
 393
 394         if (h->ae_algo->ops->add_uc_addr)
 395                 return h->ae_algo->ops->add_uc_addr(h, addr);
 396
 397         return 0;
 398 }
 399
 400 static int hns3_nic_uc_unsync(struct net_device *netdev,
 401                               const unsigned char *addr)
 402 {
 403         struct hnae3_handle *h = hns3_get_handle(netdev);
 404
 405         if (h->ae_algo->ops->rm_uc_addr)
 406                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 407
 408         return 0;
 409 }
 410
 411 static int hns3_nic_mc_sync(struct net_device *netdev,
 412                             const unsigned char *addr)
 413 {
 414         struct hnae3_handle *h = hns3_get_handle(netdev);
 415
 416         if (h->ae_algo->ops->add_mc_addr)
 417                 return h->ae_algo->ops->add_mc_addr(h, addr);
 418
 419         return 0;
 420 }
 421
 422 static int hns3_nic_mc_unsync(struct net_device *netdev,
 423                               const unsigned char *addr)
 424 {
 425         struct hnae3_handle *h = hns3_get_handle(netdev);
 426
 427         if (h->ae_algo->ops->rm_mc_addr)
 428                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 429
 430         return 0;
 431 }
 432
 433 static void hns3_nic_set_rx_mode(struct net_device *netdev)
 434 {
 435         struct hnae3_handle *h = hns3_get_handle(netdev);
 436
 437         if (h->ae_algo->ops->set_promisc_mode) {
 438                 if (netdev->flags & IFF_PROMISC)
 439                         h->ae_algo->ops->set_promisc_mode(h, true, true);
 440                 else if (netdev->flags & IFF_ALLMULTI)
 441                         h->ae_algo->ops->set_promisc_mode(h, false, true);
 442                 else
 443                         h->ae_algo->ops->set_promisc_mode(h, false, false);
 444         }
 445         if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
 446                 netdev_err(netdev, "sync uc address fail\n");
 447         if (netdev->flags & IFF_MULTICAST) {
 448                 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
 449                         netdev_err(netdev, "sync mc address fail\n");
 450
 451                 if (h->ae_algo->ops->update_mta_status)
 452                         h->ae_algo->ops->update_mta_status(h);
 453         }
 454 }
 455
 456 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 457                         u16 *mss, u32 *type_cs_vlan_tso)
 458 {
 459         u32 l4_offset, hdr_len;
 460         union l3_hdr_info l3;
 461         union l4_hdr_info l4;
 462         u32 l4_paylen;
 463         int ret;
 464
 465         if (!skb_is_gso(skb))
 466                 return 0;
 467
 468         ret = skb_cow_head(skb, 0);
 469         if (ret)
 470                 return ret;
 471
 472         l3.hdr = skb_network_header(skb);
 473         l4.hdr = skb_transport_header(skb);
 474
 475         /* Software should clear the IPv4's checksum field when tso is
 476          * needed.
 477          */
 478         if (l3.v4->version == 4)
 479                 l3.v4->check = 0;
 480
 481         /* tunnel packet.*/
 482         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 483                                          SKB_GSO_GRE_CSUM |
 484                                          SKB_GSO_UDP_TUNNEL |
 485                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 486                 if ((!(skb_shinfo(skb)->gso_type &
 487                     SKB_GSO_PARTIAL)) &&
 488                     (skb_shinfo(skb)->gso_type &
 489                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 490                         /* Software should clear the udp's checksum
 491                          * field when tso is needed.
 492                          */
 493                         l4.udp->check = 0;
 494                 }
 495                 /* reset l3&l4 pointers from outer to inner headers */
 496                 l3.hdr = skb_inner_network_header(skb);
 497                 l4.hdr = skb_inner_transport_header(skb);
 498
 499                 /* Software should clear the IPv4's checksum field when
 500                  * tso is needed.
 501                  */
 502                 if (l3.v4->version == 4)
 503                         l3.v4->check = 0;
 504         }
 505
 506         /* normal or tunnel packet*/
 507         l4_offset = l4.hdr - skb->data;
 508         hdr_len = (l4.tcp->doff * 4) + l4_offset;
 509
 510         /* remove payload length from inner pseudo checksum when tso*/
 511         l4_paylen = skb->len - l4_offset;
 512         csum_replace_by_diff(&l4.tcp->check,
 513                              (__force __wsum)htonl(l4_paylen));
 514
 515         /* find the txbd field values */
 516         *paylen = skb->len - hdr_len;
 517         hnae3_set_bit(*type_cs_vlan_tso,
 518                       HNS3_TXD_TSO_B, 1);
 519
 520         /* get MSS for TSO */
 521         *mss = skb_shinfo(skb)->gso_size;
 522
 523         return 0;
 524 }
 525
 526 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 527                                 u8 *il4_proto)
 528 {
 529         union {
 530                 struct iphdr *v4;
 531                 struct ipv6hdr *v6;
 532                 unsigned char *hdr;
 533         } l3;
 534         unsigned char *l4_hdr;
 535         unsigned char *exthdr;
 536         u8 l4_proto_tmp;
 537         __be16 frag_off;
 538
 539         /* find outer header point */
 540         l3.hdr = skb_network_header(skb);
 541         l4_hdr = skb_transport_header(skb);
 542
 543         if (skb->protocol == htons(ETH_P_IPV6)) {
 544                 exthdr = l3.hdr + sizeof(*l3.v6);
 545                 l4_proto_tmp = l3.v6->nexthdr;
 546                 if (l4_hdr != exthdr)
 547                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 548                                          &l4_proto_tmp, &frag_off);
 549         } else if (skb->protocol == htons(ETH_P_IP)) {
 550                 l4_proto_tmp = l3.v4->protocol;
 551         } else {
 552                 return -EINVAL;
 553         }
 554
 555         *ol4_proto = l4_proto_tmp;
 556
 557         /* tunnel packet */
 558         if (!skb->encapsulation) {
 559                 *il4_proto = 0;
 560                 return 0;
 561         }
 562
 563         /* find inner header point */
 564         l3.hdr = skb_inner_network_header(skb);
 565         l4_hdr = skb_inner_transport_header(skb);
 566
 567         if (l3.v6->version == 6) {
 568                 exthdr = l3.hdr + sizeof(*l3.v6);
 569                 l4_proto_tmp = l3.v6->nexthdr;
 570                 if (l4_hdr != exthdr)
 571                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 572                                          &l4_proto_tmp, &frag_off);
 573         } else if (l3.v4->version == 4) {
 574                 l4_proto_tmp = l3.v4->protocol;
 575         }
 576
 577         *il4_proto = l4_proto_tmp;
 578
 579         return 0;
 580 }
 581
 582 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
 583                                 u8 il4_proto, u32 *type_cs_vlan_tso,
 584                                 u32 *ol_type_vlan_len_msec)
 585 {
 586         union {
 587                 struct iphdr *v4;
 588                 struct ipv6hdr *v6;
 589                 unsigned char *hdr;
 590         } l3;
 591         union {
 592                 struct tcphdr *tcp;
 593                 struct udphdr *udp;
 594                 struct gre_base_hdr *gre;
 595                 unsigned char *hdr;
 596         } l4;
 597         unsigned char *l2_hdr;
 598         u8 l4_proto = ol4_proto;
 599         u32 ol2_len;
 600         u32 ol3_len;
 601         u32 ol4_len;
 602         u32 l2_len;
 603         u32 l3_len;
 604
 605         l3.hdr = skb_network_header(skb);
 606         l4.hdr = skb_transport_header(skb);
 607
 608         /* compute L2 header size for normal packet, defined in 2 Bytes */
 609         l2_len = l3.hdr - skb->data;
 610         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 611                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 612
 613         /* tunnel packet*/
 614         if (skb->encapsulation) {
 615                 /* compute OL2 header size, defined in 2 Bytes */
 616                 ol2_len = l2_len;
 617                 hnae3_set_field(*ol_type_vlan_len_msec,
 618                                 HNS3_TXD_L2LEN_M,
 619                                 HNS3_TXD_L2LEN_S, ol2_len >> 1);
 620
 621                 /* compute OL3 header size, defined in 4 Bytes */
 622                 ol3_len = l4.hdr - l3.hdr;
 623                 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
 624                                 HNS3_TXD_L3LEN_S, ol3_len >> 2);
 625
 626                 /* MAC in UDP, MAC in GRE (0x6558)*/
 627                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
 628                         /* switch MAC header ptr from outer to inner header.*/
 629                         l2_hdr = skb_inner_mac_header(skb);
 630
 631                         /* compute OL4 header size, defined in 4 Bytes. */
 632                         ol4_len = l2_hdr - l4.hdr;
 633                         hnae3_set_field(*ol_type_vlan_len_msec,
 634                                         HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
 635                                         ol4_len >> 2);
 636
 637                         /* switch IP header ptr from outer to inner header */
 638                         l3.hdr = skb_inner_network_header(skb);
 639
 640                         /* compute inner l2 header size, defined in 2 Bytes. */
 641                         l2_len = l3.hdr - l2_hdr;
 642                         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 643                                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 644                 } else {
 645                         /* skb packet types not supported by hardware,
 646                          * txbd len fild doesn't be filled.
 647                          */
 648                         return;
 649                 }
 650
 651                 /* switch L4 header pointer from outer to inner */
 652                 l4.hdr = skb_inner_transport_header(skb);
 653
 654                 l4_proto = il4_proto;
 655         }
 656
 657         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 658         l3_len = l4.hdr - l3.hdr;
 659         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
 660                         HNS3_TXD_L3LEN_S, l3_len >> 2);
 661
 662         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 663         switch (l4_proto) {
 664         case IPPROTO_TCP:
 665                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 666                                 HNS3_TXD_L4LEN_S, l4.tcp->doff);
 667                 break;
 668         case IPPROTO_SCTP:
 669                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 670                                 HNS3_TXD_L4LEN_S,
 671                                 (sizeof(struct sctphdr) >> 2));
 672                 break;
 673         case IPPROTO_UDP:
 674                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 675                                 HNS3_TXD_L4LEN_S,
 676                                 (sizeof(struct udphdr) >> 2));
 677                 break;
 678         default:
 679                 /* skb packet types not supported by hardware,
 680                  * txbd len fild doesn't be filled.
 681                  */
 682                 return;
 683         }
 684 }
 685
 686 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 687  * and it is udp packet, which has a dest port as the IANA assigned.
 688  * the hardware is expected to do the checksum offload, but the
 689  * hardware will not do the checksum offload when udp dest port is
 690  * 4789.
 691  */
 692 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 693 {
 694 #define IANA_VXLAN_PORT 4789
 695         union {
 696                 struct tcphdr *tcp;
 697                 struct udphdr *udp;
 698                 struct gre_base_hdr *gre;
 699                 unsigned char *hdr;
 700         } l4;
 701
 702         l4.hdr = skb_transport_header(skb);
 703
 704         if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
 705                 return false;
 706
 707         return true;
 708 }
 709
 710 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
 711                                    u8 il4_proto, u32 *type_cs_vlan_tso,
 712                                    u32 *ol_type_vlan_len_msec)
 713 {
 714         union {
 715                 struct iphdr *v4;
 716                 struct ipv6hdr *v6;
 717                 unsigned char *hdr;
 718         } l3;
 719         u32 l4_proto = ol4_proto;
 720
 721         l3.hdr = skb_network_header(skb);
 722
 723         /* define OL3 type and tunnel type(OL4).*/
 724         if (skb->encapsulation) {
 725                 /* define outer network header type.*/
 726                 if (skb->protocol == htons(ETH_P_IP)) {
 727                         if (skb_is_gso(skb))
 728                                 hnae3_set_field(*ol_type_vlan_len_msec,
 729                                                 HNS3_TXD_OL3T_M,
 730                                                 HNS3_TXD_OL3T_S,
 731                                                 HNS3_OL3T_IPV4_CSUM);
 732                         else
 733                                 hnae3_set_field(*ol_type_vlan_len_msec,
 734                                                 HNS3_TXD_OL3T_M,
 735                                                 HNS3_TXD_OL3T_S,
 736                                                 HNS3_OL3T_IPV4_NO_CSUM);
 737
 738                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 739                         hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
 740                                         HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
 741                 }
 742
 743                 /* define tunnel type(OL4).*/
 744                 switch (l4_proto) {
 745                 case IPPROTO_UDP:
 746                         hnae3_set_field(*ol_type_vlan_len_msec,
 747                                         HNS3_TXD_TUNTYPE_M,
 748                                         HNS3_TXD_TUNTYPE_S,
 749                                         HNS3_TUN_MAC_IN_UDP);
 750                         break;
 751                 case IPPROTO_GRE:
 752                         hnae3_set_field(*ol_type_vlan_len_msec,
 753                                         HNS3_TXD_TUNTYPE_M,
 754                                         HNS3_TXD_TUNTYPE_S,
 755                                         HNS3_TUN_NVGRE);
 756                         break;
 757                 default:
 758                         /* drop the skb tunnel packet if hardware don't support,
 759                          * because hardware can't calculate csum when TSO.
 760                          */
 761                         if (skb_is_gso(skb))
 762                                 return -EDOM;
 763
 764                         /* the stack computes the IP header already,
 765                          * driver calculate l4 checksum when not TSO.
 766                          */
 767                         return skb_checksum_help(skb);
 768                 }
 769
 770                 l3.hdr = skb_inner_network_header(skb);
 771                 l4_proto = il4_proto;
 772         }
 773
 774         if (l3.v4->version == 4) {
 775                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 776                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
 777
 778                 /* the stack computes the IP header already, the only time we
 779                  * need the hardware to recompute it is in the case of TSO.
 780                  */
 781                 if (skb_is_gso(skb))
 782                         hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 783         } else if (l3.v6->version == 6) {
 784                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 785                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
 786         }
 787
 788         switch (l4_proto) {
 789         case IPPROTO_TCP:
 790                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 791                 hnae3_set_field(*type_cs_vlan_tso,
 792                                 HNS3_TXD_L4T_M,
 793                                 HNS3_TXD_L4T_S,
 794                                 HNS3_L4T_TCP);
 795                 break;
 796         case IPPROTO_UDP:
 797                 if (hns3_tunnel_csum_bug(skb)) {
 798                         int ret = skb_put_padto(skb, HNS3_MIN_TUN_PKT_LEN);
 799
 800                         return ret ? ret : skb_checksum_help(skb);
 801                 }
 802
 803                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 804                 hnae3_set_field(*type_cs_vlan_tso,
 805                                 HNS3_TXD_L4T_M,
 806                                 HNS3_TXD_L4T_S,
 807                                 HNS3_L4T_UDP);
 808                 break;
 809         case IPPROTO_SCTP:
 810                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 811                 hnae3_set_field(*type_cs_vlan_tso,
 812                                 HNS3_TXD_L4T_M,
 813                                 HNS3_TXD_L4T_S,
 814                                 HNS3_L4T_SCTP);
 815                 break;
 816         default:
 817                 /* drop the skb tunnel packet if hardware don't support,
 818                  * because hardware can't calculate csum when TSO.
 819                  */
 820                 if (skb_is_gso(skb))
 821                         return -EDOM;
 822
 823                 /* the stack computes the IP header already,
 824                  * driver calculate l4 checksum when not TSO.
 825                  */
 826                 return skb_checksum_help(skb);
 827         }
 828
 829         return 0;
 830 }
 831
 832 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 833 {
 834         /* Config bd buffer end */
 835         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
 836                         HNS3_TXD_BDTYPE_S, 0);
 837         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
 838         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
 839         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
 840 }
 841
 842 static int hns3_fill_desc_vtags(struct sk_buff *skb,
 843                                 struct hns3_enet_ring *tx_ring,
 844                                 u32 *inner_vlan_flag,
 845                                 u32 *out_vlan_flag,
 846                                 u16 *inner_vtag,
 847                                 u16 *out_vtag)
 848 {
 849 #define HNS3_TX_VLAN_PRIO_SHIFT 13
 850
 851         if (skb->protocol == htons(ETH_P_8021Q) &&
 852             !(tx_ring->tqp->handle->kinfo.netdev->features &
 853             NETIF_F_HW_VLAN_CTAG_TX)) {
 854                 /* When HW VLAN acceleration is turned off, and the stack
 855                  * sets the protocol to 802.1q, the driver just need to
 856                  * set the protocol to the encapsulated ethertype.
 857                  */
 858                 skb->protocol = vlan_get_protocol(skb);
 859                 return 0;
 860         }
 861
 862         if (skb_vlan_tag_present(skb)) {
 863                 u16 vlan_tag;
 864
 865                 vlan_tag = skb_vlan_tag_get(skb);
 866                 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
 867
 868                 /* Based on hw strategy, use out_vtag in two layer tag case,
 869                  * and use inner_vtag in one tag case.
 870                  */
 871                 if (skb->protocol == htons(ETH_P_8021Q)) {
 872                         hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
 873                         *out_vtag = vlan_tag;
 874                 } else {
 875                         hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
 876                         *inner_vtag = vlan_tag;
 877                 }
 878         } else if (skb->protocol == htons(ETH_P_8021Q)) {
 879                 struct vlan_ethhdr *vhdr;
 880                 int rc;
 881
 882                 rc = skb_cow_head(skb, 0);
 883                 if (rc < 0)
 884                         return rc;
 885                 vhdr = (struct vlan_ethhdr *)skb->data;
 886                 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
 887                                         << HNS3_TX_VLAN_PRIO_SHIFT);
 888         }
 889
 890         skb->protocol = vlan_get_protocol(skb);
 891         return 0;
 892 }
 893
 894 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 895                           int size, dma_addr_t dma, int frag_end,
 896                           enum hns_desc_type type)
 897 {
 898         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 899         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
 900         u32 ol_type_vlan_len_msec = 0;
 901         u16 bdtp_fe_sc_vld_ra_ri = 0;
 902         u32 type_cs_vlan_tso = 0;
 903         struct sk_buff *skb;
 904         u16 inner_vtag = 0;
 905         u16 out_vtag = 0;
 906         u32 paylen = 0;
 907         u16 mss = 0;
 908         u8 ol4_proto;
 909         u8 il4_proto;
 910         int ret;
 911
 912         /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
 913         desc_cb->priv = priv;
 914         desc_cb->length = size;
 915         desc_cb->dma = dma;
 916         desc_cb->type = type;
 917
 918         /* now, fill the descriptor */
 919         desc->addr = cpu_to_le64(dma);
 920         desc->tx.send_size = cpu_to_le16((u16)size);
 921         hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
 922         desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
 923
 924         if (type == DESC_TYPE_SKB) {
 925                 skb = (struct sk_buff *)priv;
 926                 paylen = skb->len;
 927
 928                 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
 929                                            &ol_type_vlan_len_msec,
 930                                            &inner_vtag, &out_vtag);
 931                 if (unlikely(ret))
 932                         return ret;
 933
 934                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 935                         skb_reset_mac_len(skb);
 936
 937                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
 938                         if (ret)
 939                                 return ret;
 940                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
 941                                             &type_cs_vlan_tso,
 942                                             &ol_type_vlan_len_msec);
 943                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
 944                                                       &type_cs_vlan_tso,
 945                                                       &ol_type_vlan_len_msec);
 946                         if (ret)
 947                                 return ret;
 948
 949                         ret = hns3_set_tso(skb, &paylen, &mss,
 950                                            &type_cs_vlan_tso);
 951                         if (ret)
 952                                 return ret;
 953                 }
 954
 955                 /* Set txbd */
 956                 desc->tx.ol_type_vlan_len_msec =
 957                         cpu_to_le32(ol_type_vlan_len_msec);
 958                 desc->tx.type_cs_vlan_tso_len =
 959                         cpu_to_le32(type_cs_vlan_tso);
 960                 desc->tx.paylen = cpu_to_le32(paylen);
 961                 desc->tx.mss = cpu_to_le16(mss);
 962                 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
 963                 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
 964         }
 965
 966         /* move ring pointer to next.*/
 967         ring_ptr_move_fw(ring, next_to_use);
 968
 969         return 0;
 970 }
 971
 972 static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
 973                               int size, dma_addr_t dma, int frag_end,
 974                               enum hns_desc_type type)
 975 {
 976         unsigned int frag_buf_num;
 977         unsigned int k;
 978         int sizeoflast;
 979         int ret;
 980
 981         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 982         sizeoflast = size % HNS3_MAX_BD_SIZE;
 983         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
 984
 985         /* When the frag size is bigger than hardware, split this frag */
 986         for (k = 0; k < frag_buf_num; k++) {
 987                 ret = hns3_fill_desc(ring, priv,
 988                                      (k == frag_buf_num - 1) ?
 989                                 sizeoflast : HNS3_MAX_BD_SIZE,
 990                                 dma + HNS3_MAX_BD_SIZE * k,
 991                                 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
 992                                 (type == DESC_TYPE_SKB && !k) ?
 993                                         DESC_TYPE_SKB : DESC_TYPE_PAGE);
 994                 if (ret)
 995                         return ret;
 996         }
 997
 998         return 0;
 999 }
1000
1001 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1002                                    struct hns3_enet_ring *ring)
1003 {
1004         struct sk_buff *skb = *out_skb;
1005         struct skb_frag_struct *frag;
1006         int bdnum_for_frag;
1007         int frag_num;
1008         int buf_num;
1009         int size;
1010         int i;
1011
1012         size = skb_headlen(skb);
1013         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1014
1015         frag_num = skb_shinfo(skb)->nr_frags;
1016         for (i = 0; i < frag_num; i++) {
1017                 frag = &skb_shinfo(skb)->frags[i];
1018                 size = skb_frag_size(frag);
1019                 bdnum_for_frag =
1020                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1021                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1022                         return -ENOMEM;
1023
1024                 buf_num += bdnum_for_frag;
1025         }
1026
1027         if (buf_num > ring_space(ring))
1028                 return -EBUSY;
1029
1030         *bnum = buf_num;
1031         return 0;
1032 }
1033
1034 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1035                                   struct hns3_enet_ring *ring)
1036 {
1037         struct sk_buff *skb = *out_skb;
1038         int buf_num;
1039
1040         /* No. of segments (plus a header) */
1041         buf_num = skb_shinfo(skb)->nr_frags + 1;
1042
1043         if (buf_num > ring_space(ring))
1044                 return -EBUSY;
1045
1046         *bnum = buf_num;
1047
1048         return 0;
1049 }
1050
1051 static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
1052 {
1053         struct device *dev = ring_to_dev(ring);
1054         unsigned int i;
1055
1056         for (i = 0; i < ring->desc_num; i++) {
1057                 /* check if this is where we started */
1058                 if (ring->next_to_use == next_to_use_orig)
1059                         break;
1060
1061                 /* unmap the descriptor dma address */
1062                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1063                         dma_unmap_single(dev,
1064                                          ring->desc_cb[ring->next_to_use].dma,
1065                                         ring->desc_cb[ring->next_to_use].length,
1066                                         DMA_TO_DEVICE);
1067                 else
1068                         dma_unmap_page(dev,
1069                                        ring->desc_cb[ring->next_to_use].dma,
1070                                        ring->desc_cb[ring->next_to_use].length,
1071                                        DMA_TO_DEVICE);
1072
1073                 /* rollback one */
1074                 ring_ptr_move_bw(ring, next_to_use);
1075         }
1076 }
1077
1078 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1079 {
1080         struct hns3_nic_priv *priv = netdev_priv(netdev);
1081         struct hns3_nic_ring_data *ring_data =
1082                 &tx_ring_data(priv, skb->queue_mapping);
1083         struct hns3_enet_ring *ring = ring_data->ring;
1084         struct device *dev = priv->dev;
1085         struct netdev_queue *dev_queue;
1086         struct skb_frag_struct *frag;
1087         int next_to_use_head;
1088         int next_to_use_frag;
1089         dma_addr_t dma;
1090         int buf_num;
1091         int seg_num;
1092         int size;
1093         int ret;
1094         int i;
1095
1096         /* Prefetch the data used later */
1097         prefetch(skb->data);
1098
1099         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1100         case -EBUSY:
1101                 u64_stats_update_begin(&ring->syncp);
1102                 ring->stats.tx_busy++;
1103                 u64_stats_update_end(&ring->syncp);
1104
1105                 goto out_net_tx_busy;
1106         case -ENOMEM:
1107                 u64_stats_update_begin(&ring->syncp);
1108                 ring->stats.sw_err_cnt++;
1109                 u64_stats_update_end(&ring->syncp);
1110                 netdev_err(netdev, "no memory to xmit!\n");
1111
1112                 goto out_err_tx_ok;
1113         default:
1114                 break;
1115         }
1116
1117         /* No. of segments (plus a header) */
1118         seg_num = skb_shinfo(skb)->nr_frags + 1;
1119         /* Fill the first part */
1120         size = skb_headlen(skb);
1121
1122         next_to_use_head = ring->next_to_use;
1123
1124         dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1125         if (dma_mapping_error(dev, dma)) {
1126                 netdev_err(netdev, "TX head DMA map failed\n");
1127                 ring->stats.sw_err_cnt++;
1128                 goto out_err_tx_ok;
1129         }
1130
1131         ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
1132                            DESC_TYPE_SKB);
1133         if (ret)
1134                 goto head_dma_map_err;
1135
1136         next_to_use_frag = ring->next_to_use;
1137         /* Fill the fragments */
1138         for (i = 1; i < seg_num; i++) {
1139                 frag = &skb_shinfo(skb)->frags[i - 1];
1140                 size = skb_frag_size(frag);
1141                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1142                 if (dma_mapping_error(dev, dma)) {
1143                         netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
1144                         ring->stats.sw_err_cnt++;
1145                         goto frag_dma_map_err;
1146                 }
1147                 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
1148                                     seg_num - 1 == i ? 1 : 0,
1149                                     DESC_TYPE_PAGE);
1150
1151                 if (ret)
1152                         goto frag_dma_map_err;
1153         }
1154
1155         /* Complete translate all packets */
1156         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1157         netdev_tx_sent_queue(dev_queue, skb->len);
1158
1159         wmb(); /* Commit all data before submit */
1160
1161         hnae3_queue_xmit(ring->tqp, buf_num);
1162
1163         return NETDEV_TX_OK;
1164
1165 frag_dma_map_err:
1166         hns_nic_dma_unmap(ring, next_to_use_frag);
1167
1168 head_dma_map_err:
1169         hns_nic_dma_unmap(ring, next_to_use_head);
1170
1171 out_err_tx_ok:
1172         dev_kfree_skb_any(skb);
1173         return NETDEV_TX_OK;
1174
1175 out_net_tx_busy:
1176         netif_stop_subqueue(netdev, ring_data->queue_index);
1177         smp_mb(); /* Commit all data before submit */
1178
1179         return NETDEV_TX_BUSY;
1180 }
1181
1182 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1183 {
1184         struct hnae3_handle *h = hns3_get_handle(netdev);
1185         struct sockaddr *mac_addr = p;
1186         int ret;
1187
1188         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1189                 return -EADDRNOTAVAIL;
1190
1191         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1192                 netdev_info(netdev, "already using mac address %pM\n",
1193                             mac_addr->sa_data);
1194                 return 0;
1195         }
1196
1197         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1198         if (ret) {
1199                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1200                 return ret;
1201         }
1202
1203         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1204
1205         return 0;
1206 }
1207
1208 static int hns3_nic_set_features(struct net_device *netdev,
1209                                  netdev_features_t features)
1210 {
1211         netdev_features_t changed = netdev->features ^ features;
1212         struct hns3_nic_priv *priv = netdev_priv(netdev);
1213         struct hnae3_handle *h = priv->ae_handle;
1214         int ret;
1215
1216         if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1217                 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1218                         priv->ops.fill_desc = hns3_fill_desc_tso;
1219                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1220                 } else {
1221                         priv->ops.fill_desc = hns3_fill_desc;
1222                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1223                 }
1224         }
1225
1226         if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1227             h->ae_algo->ops->enable_vlan_filter) {
1228                 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1229                         h->ae_algo->ops->enable_vlan_filter(h, true);
1230                 else
1231                         h->ae_algo->ops->enable_vlan_filter(h, false);
1232         }
1233
1234         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1235             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1236                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1237                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1238                 else
1239                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1240
1241                 if (ret)
1242                         return ret;
1243         }
1244
1245         netdev->features = features;
1246         return 0;
1247 }
1248
1249 static void hns3_nic_get_stats64(struct net_device *netdev,
1250                                  struct rtnl_link_stats64 *stats)
1251 {
1252         struct hns3_nic_priv *priv = netdev_priv(netdev);
1253         int queue_num = priv->ae_handle->kinfo.num_tqps;
1254         struct hnae3_handle *handle = priv->ae_handle;
1255         struct hns3_enet_ring *ring;
1256         unsigned int start;
1257         unsigned int idx;
1258         u64 tx_bytes = 0;
1259         u64 rx_bytes = 0;
1260         u64 tx_pkts = 0;
1261         u64 rx_pkts = 0;
1262         u64 tx_drop = 0;
1263         u64 rx_drop = 0;
1264
1265         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1266                 return;
1267
1268         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1269
1270         for (idx = 0; idx < queue_num; idx++) {
1271                 /* fetch the tx stats */
1272                 ring = priv->ring_data[idx].ring;
1273                 do {
1274                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1275                         tx_bytes += ring->stats.tx_bytes;
1276                         tx_pkts += ring->stats.tx_pkts;
1277                         tx_drop += ring->stats.tx_busy;
1278                         tx_drop += ring->stats.sw_err_cnt;
1279                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1280
1281                 /* fetch the rx stats */
1282                 ring = priv->ring_data[idx + queue_num].ring;
1283                 do {
1284                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1285                         rx_bytes += ring->stats.rx_bytes;
1286                         rx_pkts += ring->stats.rx_pkts;
1287                         rx_drop += ring->stats.non_vld_descs;
1288                         rx_drop += ring->stats.err_pkt_len;
1289                         rx_drop += ring->stats.l2_err;
1290                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1291         }
1292
1293         stats->tx_bytes = tx_bytes;
1294         stats->tx_packets = tx_pkts;
1295         stats->rx_bytes = rx_bytes;
1296         stats->rx_packets = rx_pkts;
1297
1298         stats->rx_errors = netdev->stats.rx_errors;
1299         stats->multicast = netdev->stats.multicast;
1300         stats->rx_length_errors = netdev->stats.rx_length_errors;
1301         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1302         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1303
1304         stats->tx_errors = netdev->stats.tx_errors;
1305         stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1306         stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1307         stats->collisions = netdev->stats.collisions;
1308         stats->rx_over_errors = netdev->stats.rx_over_errors;
1309         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1310         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1311         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1312         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1313         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1314         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1315         stats->tx_window_errors = netdev->stats.tx_window_errors;
1316         stats->rx_compressed = netdev->stats.rx_compressed;
1317         stats->tx_compressed = netdev->stats.tx_compressed;
1318 }
1319
1320 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1321 {
1322         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1323         struct hnae3_handle *h = hns3_get_handle(netdev);
1324         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1325         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1326         u8 tc = mqprio_qopt->qopt.num_tc;
1327         u16 mode = mqprio_qopt->mode;
1328         u8 hw = mqprio_qopt->qopt.hw;
1329         bool if_running;
1330         int ret;
1331
1332         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1333                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1334                 return -EOPNOTSUPP;
1335
1336         if (tc > HNAE3_MAX_TC)
1337                 return -EINVAL;
1338
1339         if (!netdev)
1340                 return -EINVAL;
1341
1342         if_running = netif_running(netdev);
1343         if (if_running) {
1344                 hns3_nic_net_stop(netdev);
1345                 msleep(100);
1346         }
1347
1348         ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1349                 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1350         if (ret)
1351                 goto out;
1352
1353         ret = hns3_nic_set_real_num_queue(netdev);
1354
1355 out:
1356         if (if_running)
1357                 hns3_nic_net_open(netdev);
1358
1359         return ret;
1360 }
1361
1362 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1363                              void *type_data)
1364 {
1365         if (type != TC_SETUP_QDISC_MQPRIO)
1366                 return -EOPNOTSUPP;
1367
1368         return hns3_setup_tc(dev, type_data);
1369 }
1370
1371 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1372                                 __be16 proto, u16 vid)
1373 {
1374         struct hnae3_handle *h = hns3_get_handle(netdev);
1375         struct hns3_nic_priv *priv = netdev_priv(netdev);
1376         int ret = -EIO;
1377
1378         if (h->ae_algo->ops->set_vlan_filter)
1379                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1380
1381         if (!ret)
1382                 set_bit(vid, priv->active_vlans);
1383
1384         return ret;
1385 }
1386
1387 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1388                                  __be16 proto, u16 vid)
1389 {
1390         struct hnae3_handle *h = hns3_get_handle(netdev);
1391         struct hns3_nic_priv *priv = netdev_priv(netdev);
1392         int ret = -EIO;
1393
1394         if (h->ae_algo->ops->set_vlan_filter)
1395                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1396
1397         if (!ret)
1398                 clear_bit(vid, priv->active_vlans);
1399
1400         return ret;
1401 }
1402
1403 static void hns3_restore_vlan(struct net_device *netdev)
1404 {
1405         struct hns3_nic_priv *priv = netdev_priv(netdev);
1406         u16 vid;
1407         int ret;
1408
1409         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1410                 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1411                 if (ret)
1412                         netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n",
1413                                     vid, ret);
1414         }
1415 }
1416
1417 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1418                                 u8 qos, __be16 vlan_proto)
1419 {
1420         struct hnae3_handle *h = hns3_get_handle(netdev);
1421         int ret = -EIO;
1422
1423         if (h->ae_algo->ops->set_vf_vlan_filter)
1424                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1425                                                    qos, vlan_proto);
1426
1427         return ret;
1428 }
1429
1430 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1431 {
1432         struct hnae3_handle *h = hns3_get_handle(netdev);
1433         bool if_running = netif_running(netdev);
1434         int ret;
1435
1436         if (!h->ae_algo->ops->set_mtu)
1437                 return -EOPNOTSUPP;
1438
1439         /* if this was called with netdev up then bring netdevice down */
1440         if (if_running) {
1441                 (void)hns3_nic_net_stop(netdev);
1442                 msleep(100);
1443         }
1444
1445         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1446         if (ret)
1447                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1448                            ret);
1449         else
1450                 netdev->mtu = new_mtu;
1451
1452         /* if the netdev was running earlier, bring it up again */
1453         if (if_running && hns3_nic_net_open(netdev))
1454                 ret = -EINVAL;
1455
1456         return ret;
1457 }
1458
1459 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1460 {
1461         struct hns3_nic_priv *priv = netdev_priv(ndev);
1462         struct hns3_enet_ring *tx_ring = NULL;
1463         int timeout_queue = 0;
1464         int hw_head, hw_tail;
1465         int i;
1466
1467         /* Find the stopped queue the same way the stack does */
1468         for (i = 0; i < ndev->num_tx_queues; i++) {
1469                 struct netdev_queue *q;
1470                 unsigned long trans_start;
1471
1472                 q = netdev_get_tx_queue(ndev, i);
1473                 trans_start = q->trans_start;
1474                 if (netif_xmit_stopped(q) &&
1475                     time_after(jiffies,
1476                                (trans_start + ndev->watchdog_timeo))) {
1477                         timeout_queue = i;
1478                         netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
1479                                     q->state,
1480                                     jiffies_to_msecs(jiffies - trans_start));
1481                         break;
1482                 }
1483         }
1484
1485         if (i == ndev->num_tx_queues) {
1486                 netdev_info(ndev,
1487                             "no netdev TX timeout queue found, timeout count: %llu\n",
1488                             priv->tx_timeout_count);
1489                 return false;
1490         }
1491
1492         tx_ring = priv->ring_data[timeout_queue].ring;
1493
1494         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1495                                 HNS3_RING_TX_RING_HEAD_REG);
1496         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1497                                 HNS3_RING_TX_RING_TAIL_REG);
1498         netdev_info(ndev,
1499                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1500                     priv->tx_timeout_count,
1501                     timeout_queue,
1502                     tx_ring->next_to_use,
1503                     tx_ring->next_to_clean,
1504                     hw_head,
1505                     hw_tail,
1506                     readl(tx_ring->tqp_vector->mask_addr));
1507
1508         return true;
1509 }
1510
1511 static void hns3_nic_net_timeout(struct net_device *ndev)
1512 {
1513         struct hns3_nic_priv *priv = netdev_priv(ndev);
1514         struct hnae3_handle *h = priv->ae_handle;
1515
1516         if (!hns3_get_tx_timeo_queue_info(ndev))
1517                 return;
1518
1519         priv->tx_timeout_count++;
1520
1521         if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
1522                 return;
1523
1524         /* request the reset */
1525         if (h->ae_algo->ops->reset_event)
1526                 h->ae_algo->ops->reset_event(h);
1527 }
1528
1529 static const struct net_device_ops hns3_nic_netdev_ops = {
1530         .ndo_open               = hns3_nic_net_open,
1531         .ndo_stop               = hns3_nic_net_stop,
1532         .ndo_start_xmit         = hns3_nic_net_xmit,
1533         .ndo_tx_timeout         = hns3_nic_net_timeout,
1534         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1535         .ndo_change_mtu         = hns3_nic_change_mtu,
1536         .ndo_set_features       = hns3_nic_set_features,
1537         .ndo_get_stats64        = hns3_nic_get_stats64,
1538         .ndo_setup_tc           = hns3_nic_setup_tc,
1539         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1540         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1541         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1542         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1543 };
1544
1545 static bool hns3_is_phys_func(struct pci_dev *pdev)
1546 {
1547         u32 dev_id = pdev->device;
1548
1549         switch (dev_id) {
1550         case HNAE3_DEV_ID_GE:
1551         case HNAE3_DEV_ID_25GE:
1552         case HNAE3_DEV_ID_25GE_RDMA:
1553         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1554         case HNAE3_DEV_ID_50GE_RDMA:
1555         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1556         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1557                 return true;
1558         case HNAE3_DEV_ID_100G_VF:
1559         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1560                 return false;
1561         default:
1562                 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1563                          dev_id);
1564         }
1565
1566         return false;
1567 }
1568
1569 static void hns3_disable_sriov(struct pci_dev *pdev)
1570 {
1571         /* If our VFs are assigned we cannot shut down SR-IOV
1572          * without causing issues, so just leave the hardware
1573          * available but disabled
1574          */
1575         if (pci_vfs_assigned(pdev)) {
1576                 dev_warn(&pdev->dev,
1577                          "disabling driver while VFs are assigned\n");
1578                 return;
1579         }
1580
1581         pci_disable_sriov(pdev);
1582 }
1583
1584 /* hns3_probe - Device initialization routine
1585  * @pdev: PCI device information struct
1586  * @ent: entry in hns3_pci_tbl
1587  *
1588  * hns3_probe initializes a PF identified by a pci_dev structure.
1589  * The OS initialization, configuring of the PF private structure,
1590  * and a hardware reset occur.
1591  *
1592  * Returns 0 on success, negative on failure
1593  */
1594 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1595 {
1596         struct hnae3_ae_dev *ae_dev;
1597         int ret;
1598
1599         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1600                               GFP_KERNEL);
1601         if (!ae_dev) {
1602                 ret = -ENOMEM;
1603                 return ret;
1604         }
1605
1606         ae_dev->pdev = pdev;
1607         ae_dev->flag = ent->driver_data;
1608         ae_dev->dev_type = HNAE3_DEV_KNIC;
1609         pci_set_drvdata(pdev, ae_dev);
1610
1611         ret = hnae3_register_ae_dev(ae_dev);
1612         if (ret) {
1613                 devm_kfree(&pdev->dev, ae_dev);
1614                 pci_set_drvdata(pdev, NULL);
1615         }
1616
1617         return ret;
1618 }
1619
1620 /* hns3_remove - Device removal routine
1621  * @pdev: PCI device information struct
1622  */
1623 static void hns3_remove(struct pci_dev *pdev)
1624 {
1625         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1626
1627         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1628                 hns3_disable_sriov(pdev);
1629
1630         hnae3_unregister_ae_dev(ae_dev);
1631         pci_set_drvdata(pdev, NULL);
1632 }
1633
1634 /**
1635  * hns3_pci_sriov_configure
1636  * @pdev: pointer to a pci_dev structure
1637  * @num_vfs: number of VFs to allocate
1638  *
1639  * Enable or change the number of VFs. Called when the user updates the number
1640  * of VFs in sysfs.
1641  **/
1642 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1643 {
1644         int ret;
1645
1646         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1647                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1648                 return -EINVAL;
1649         }
1650
1651         if (num_vfs) {
1652                 ret = pci_enable_sriov(pdev, num_vfs);
1653                 if (ret)
1654                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1655                 else
1656                         return num_vfs;
1657         } else if (!pci_vfs_assigned(pdev)) {
1658                 pci_disable_sriov(pdev);
1659         } else {
1660                 dev_warn(&pdev->dev,
1661                          "Unable to free VFs because some are assigned to VMs.\n");
1662         }
1663
1664         return 0;
1665 }
1666
1667 static struct pci_driver hns3_driver = {
1668         .name     = hns3_driver_name,
1669         .id_table = hns3_pci_tbl,
1670         .probe    = hns3_probe,
1671         .remove   = hns3_remove,
1672         .sriov_configure = hns3_pci_sriov_configure,
1673 };
1674
1675 /* set default feature to hns3 */
1676 static void hns3_set_default_feature(struct net_device *netdev)
1677 {
1678         struct hnae3_handle *h = hns3_get_handle(netdev);
1679         struct pci_dev *pdev = h->pdev;
1680
1681         netdev->priv_flags |= IFF_UNICAST_FLT;
1682
1683         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1684                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1685                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1686                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1687                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1688
1689         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1690
1691         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1692
1693         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1694                 NETIF_F_HW_VLAN_CTAG_FILTER |
1695                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1696                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1697                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1698                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1699                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1700
1701         netdev->vlan_features |=
1702                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1703                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1704                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1705                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1706                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1707
1708         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1709                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1710                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1711                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1712                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1713                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1714
1715         if (pdev->revision != 0x20)
1716                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1717 }
1718
1719 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1720                              struct hns3_desc_cb *cb)
1721 {
1722         unsigned int order = hnae3_page_order(ring);
1723         struct page *p;
1724
1725         p = dev_alloc_pages(order);
1726         if (!p)
1727                 return -ENOMEM;
1728
1729         cb->priv = p;
1730         cb->page_offset = 0;
1731         cb->reuse_flag = 0;
1732         cb->buf  = page_address(p);
1733         cb->length = hnae3_page_size(ring);
1734         cb->type = DESC_TYPE_PAGE;
1735
1736         return 0;
1737 }
1738
1739 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1740                              struct hns3_desc_cb *cb)
1741 {
1742         if (cb->type == DESC_TYPE_SKB)
1743                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1744         else if (!HNAE3_IS_TX_RING(ring))
1745                 put_page((struct page *)cb->priv);
1746         memset(cb, 0, sizeof(*cb));
1747 }
1748
1749 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1750 {
1751         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1752                                cb->length, ring_to_dma_dir(ring));
1753
1754         if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1755                 return -EIO;
1756
1757         return 0;
1758 }
1759
1760 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1761                               struct hns3_desc_cb *cb)
1762 {
1763         if (cb->type == DESC_TYPE_SKB)
1764                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1765                                  ring_to_dma_dir(ring));
1766         else
1767                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1768                                ring_to_dma_dir(ring));
1769 }
1770
1771 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1772 {
1773         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1774         ring->desc[i].addr = 0;
1775 }
1776
1777 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1778 {
1779         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1780
1781         if (!ring->desc_cb[i].dma)
1782                 return;
1783
1784         hns3_buffer_detach(ring, i);
1785         hns3_free_buffer(ring, cb);
1786 }
1787
1788 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1789 {
1790         int i;
1791
1792         for (i = 0; i < ring->desc_num; i++)
1793                 hns3_free_buffer_detach(ring, i);
1794 }
1795
1796 /* free desc along with its attached buffer */
1797 static void hns3_free_desc(struct hns3_enet_ring *ring)
1798 {
1799         int size = ring->desc_num * sizeof(ring->desc[0]);
1800
1801         hns3_free_buffers(ring);
1802
1803         if (ring->desc) {
1804                 dma_free_coherent(ring_to_dev(ring), size,
1805                                   ring->desc, ring->desc_dma_addr);
1806                 ring->desc = NULL;
1807         }
1808 }
1809
1810 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1811 {
1812         int size = ring->desc_num * sizeof(ring->desc[0]);
1813
1814         ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
1815                                          &ring->desc_dma_addr,
1816                                          GFP_KERNEL);
1817         if (!ring->desc)
1818                 return -ENOMEM;
1819
1820         return 0;
1821 }
1822
1823 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1824                                    struct hns3_desc_cb *cb)
1825 {
1826         int ret;
1827
1828         ret = hns3_alloc_buffer(ring, cb);
1829         if (ret)
1830                 goto out;
1831
1832         ret = hns3_map_buffer(ring, cb);
1833         if (ret)
1834                 goto out_with_buf;
1835
1836         return 0;
1837
1838 out_with_buf:
1839         hns3_free_buffer(ring, cb);
1840 out:
1841         return ret;
1842 }
1843
1844 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1845 {
1846         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1847
1848         if (ret)
1849                 return ret;
1850
1851         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1852
1853         return 0;
1854 }
1855
1856 /* Allocate memory for raw pkg, and map with dma */
1857 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1858 {
1859         int i, j, ret;
1860
1861         for (i = 0; i < ring->desc_num; i++) {
1862                 ret = hns3_alloc_buffer_attach(ring, i);
1863                 if (ret)
1864                         goto out_buffer_fail;
1865         }
1866
1867         return 0;
1868
1869 out_buffer_fail:
1870         for (j = i - 1; j >= 0; j--)
1871                 hns3_free_buffer_detach(ring, j);
1872         return ret;
1873 }
1874
1875 /* detach a in-used buffer and replace with a reserved one  */
1876 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1877                                 struct hns3_desc_cb *res_cb)
1878 {
1879         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1880         ring->desc_cb[i] = *res_cb;
1881         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1882         ring->desc[i].rx.bd_base_info = 0;
1883 }
1884
1885 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1886 {
1887         ring->desc_cb[i].reuse_flag = 0;
1888         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1889                 + ring->desc_cb[i].page_offset);
1890         ring->desc[i].rx.bd_base_info = 0;
1891 }
1892
1893 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1894                                       int *pkts)
1895 {
1896         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1897
1898         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1899         (*bytes) += desc_cb->length;
1900         /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
1901         hns3_free_buffer_detach(ring, ring->next_to_clean);
1902
1903         ring_ptr_move_fw(ring, next_to_clean);
1904 }
1905
1906 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1907 {
1908         int u = ring->next_to_use;
1909         int c = ring->next_to_clean;
1910
1911         if (unlikely(h > ring->desc_num))
1912                 return 0;
1913
1914         return u > c ? (h > c && h <= u) : (h > c || h <= u);
1915 }
1916
1917 bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
1918 {
1919         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1920         struct hns3_nic_priv *priv = netdev_priv(netdev);
1921         struct netdev_queue *dev_queue;
1922         int bytes, pkts;
1923         int head;
1924
1925         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1926         rmb(); /* Make sure head is ready before touch any data */
1927
1928         if (is_ring_empty(ring) || head == ring->next_to_clean)
1929                 return true; /* no data to poll */
1930
1931         if (unlikely(!is_valid_clean_head(ring, head))) {
1932                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1933                            ring->next_to_use, ring->next_to_clean);
1934
1935                 u64_stats_update_begin(&ring->syncp);
1936                 ring->stats.io_err_cnt++;
1937                 u64_stats_update_end(&ring->syncp);
1938                 return true;
1939         }
1940
1941         bytes = 0;
1942         pkts = 0;
1943         while (head != ring->next_to_clean && budget) {
1944                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1945                 /* Issue prefetch for next Tx descriptor */
1946                 prefetch(&ring->desc_cb[ring->next_to_clean]);
1947                 budget--;
1948         }
1949
1950         ring->tqp_vector->tx_group.total_bytes += bytes;
1951         ring->tqp_vector->tx_group.total_packets += pkts;
1952
1953         u64_stats_update_begin(&ring->syncp);
1954         ring->stats.tx_bytes += bytes;
1955         ring->stats.tx_pkts += pkts;
1956         u64_stats_update_end(&ring->syncp);
1957
1958         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1959         netdev_tx_completed_queue(dev_queue, pkts, bytes);
1960
1961         if (unlikely(pkts && netif_carrier_ok(netdev) &&
1962                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1963                 /* Make sure that anybody stopping the queue after this
1964                  * sees the new next_to_clean.
1965                  */
1966                 smp_mb();
1967                 if (netif_tx_queue_stopped(dev_queue) &&
1968                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1969                         netif_tx_wake_queue(dev_queue);
1970                         ring->stats.restart_queue++;
1971                 }
1972         }
1973
1974         return !!budget;
1975 }
1976
1977 static int hns3_desc_unused(struct hns3_enet_ring *ring)
1978 {
1979         int ntc = ring->next_to_clean;
1980         int ntu = ring->next_to_use;
1981
1982         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1983 }
1984
1985 static void
1986 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1987 {
1988         struct hns3_desc_cb *desc_cb;
1989         struct hns3_desc_cb res_cbs;
1990         int i, ret;
1991
1992         for (i = 0; i < cleand_count; i++) {
1993                 desc_cb = &ring->desc_cb[ring->next_to_use];
1994                 if (desc_cb->reuse_flag) {
1995                         u64_stats_update_begin(&ring->syncp);
1996                         ring->stats.reuse_pg_cnt++;
1997                         u64_stats_update_end(&ring->syncp);
1998
1999                         hns3_reuse_buffer(ring, ring->next_to_use);
2000                 } else {
2001                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
2002                         if (ret) {
2003                                 u64_stats_update_begin(&ring->syncp);
2004                                 ring->stats.sw_err_cnt++;
2005                                 u64_stats_update_end(&ring->syncp);
2006
2007                                 netdev_err(ring->tqp->handle->kinfo.netdev,
2008                                            "hnae reserve buffer map failed.\n");
2009                                 break;
2010                         }
2011                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2012                 }
2013
2014                 ring_ptr_move_fw(ring, next_to_use);
2015         }
2016
2017         wmb(); /* Make all data has been write before submit */
2018         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2019 }
2020
2021 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2022                                 struct hns3_enet_ring *ring, int pull_len,
2023                                 struct hns3_desc_cb *desc_cb)
2024 {
2025         struct hns3_desc *desc;
2026         u32 truesize;
2027         int size;
2028         int last_offset;
2029         bool twobufs;
2030
2031         twobufs = ((PAGE_SIZE < 8192) &&
2032                 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2033
2034         desc = &ring->desc[ring->next_to_clean];
2035         size = le16_to_cpu(desc->rx.size);
2036
2037         truesize = hnae3_buf_size(ring);
2038
2039         if (!twobufs)
2040                 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2041
2042         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2043                         size - pull_len, truesize);
2044
2045          /* Avoid re-using remote pages,flag default unreuse */
2046         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2047                 return;
2048
2049         if (twobufs) {
2050                 /* If we are only owner of page we can reuse it */
2051                 if (likely(page_count(desc_cb->priv) == 1)) {
2052                         /* Flip page offset to other buffer */
2053                         desc_cb->page_offset ^= truesize;
2054
2055                         desc_cb->reuse_flag = 1;
2056                         /* bump ref count on page before it is given*/
2057                         get_page(desc_cb->priv);
2058                 }
2059                 return;
2060         }
2061
2062         /* Move offset up to the next cache line */
2063         desc_cb->page_offset += truesize;
2064
2065         if (desc_cb->page_offset <= last_offset) {
2066                 desc_cb->reuse_flag = 1;
2067                 /* Bump ref count on page before it is given*/
2068                 get_page(desc_cb->priv);
2069         }
2070 }
2071
2072 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2073                              struct hns3_desc *desc)
2074 {
2075         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2076         int l3_type, l4_type;
2077         u32 bd_base_info;
2078         int ol4_type;
2079         u32 l234info;
2080
2081         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2082         l234info = le32_to_cpu(desc->rx.l234_info);
2083
2084         skb->ip_summed = CHECKSUM_NONE;
2085
2086         skb_checksum_none_assert(skb);
2087
2088         if (!(netdev->features & NETIF_F_RXCSUM))
2089                 return;
2090
2091         /* check if hardware has done checksum */
2092         if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2093                 return;
2094
2095         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2096                      hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2097                      hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2098                      hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2099                 netdev_err(netdev, "L3/L4 error pkt\n");
2100                 u64_stats_update_begin(&ring->syncp);
2101                 ring->stats.l3l4_csum_err++;
2102                 u64_stats_update_end(&ring->syncp);
2103
2104                 return;
2105         }
2106
2107         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2108                                   HNS3_RXD_L3ID_S);
2109         l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2110                                   HNS3_RXD_L4ID_S);
2111
2112         ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2113                                    HNS3_RXD_OL4ID_S);
2114         switch (ol4_type) {
2115         case HNS3_OL4_TYPE_MAC_IN_UDP:
2116         case HNS3_OL4_TYPE_NVGRE:
2117                 skb->csum_level = 1;
2118                 /* fall through */
2119         case HNS3_OL4_TYPE_NO_TUN:
2120                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2121                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2122                      l3_type == HNS3_L3_TYPE_IPV6) &&
2123                     (l4_type == HNS3_L4_TYPE_UDP ||
2124                      l4_type == HNS3_L4_TYPE_TCP ||
2125                      l4_type == HNS3_L4_TYPE_SCTP))
2126                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2127                 break;
2128         }
2129 }
2130
2131 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2132 {
2133         napi_gro_receive(&ring->tqp_vector->napi, skb);
2134 }
2135
2136 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2137                                 struct hns3_desc *desc, u32 l234info,
2138                                 u16 *vlan_tag)
2139 {
2140         struct pci_dev *pdev = ring->tqp->handle->pdev;
2141
2142         if (pdev->revision == 0x20) {
2143                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2144                 if (!(*vlan_tag & VLAN_VID_MASK))
2145                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2146
2147                 return (*vlan_tag != 0);
2148         }
2149
2150 #define HNS3_STRP_OUTER_VLAN    0x1
2151 #define HNS3_STRP_INNER_VLAN    0x2
2152
2153         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2154                                 HNS3_RXD_STRP_TAGP_S)) {
2155         case HNS3_STRP_OUTER_VLAN:
2156                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2157                 return true;
2158         case HNS3_STRP_INNER_VLAN:
2159                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2160                 return true;
2161         default:
2162                 return false;
2163         }
2164 }
2165
2166 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2167                              struct sk_buff **out_skb, int *out_bnum)
2168 {
2169         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2170         struct hns3_desc_cb *desc_cb;
2171         struct hns3_desc *desc;
2172         struct sk_buff *skb;
2173         unsigned char *va;
2174         u32 bd_base_info;
2175         int pull_len;
2176         u32 l234info;
2177         int length;
2178         int bnum;
2179
2180         desc = &ring->desc[ring->next_to_clean];
2181         desc_cb = &ring->desc_cb[ring->next_to_clean];
2182
2183         prefetch(desc);
2184
2185         length = le16_to_cpu(desc->rx.size);
2186         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2187
2188         /* Check valid BD */
2189         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2190                 return -EFAULT;
2191
2192         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2193
2194         /* Prefetch first cache line of first page
2195          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2196          * line size is 64B so need to prefetch twice to make it 128B. But in
2197          * actual we can have greater size of caches with 128B Level 1 cache
2198          * lines. In such a case, single fetch would suffice to cache in the
2199          * relevant part of the header.
2200          */
2201         prefetch(va);
2202 #if L1_CACHE_BYTES < 128
2203         prefetch(va + L1_CACHE_BYTES);
2204 #endif
2205
2206         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2207                                         HNS3_RX_HEAD_SIZE);
2208         if (unlikely(!skb)) {
2209                 netdev_err(netdev, "alloc rx skb fail\n");
2210
2211                 u64_stats_update_begin(&ring->syncp);
2212                 ring->stats.sw_err_cnt++;
2213                 u64_stats_update_end(&ring->syncp);
2214
2215                 return -ENOMEM;
2216         }
2217
2218         prefetchw(skb->data);
2219
2220         bnum = 1;
2221         if (length <= HNS3_RX_HEAD_SIZE) {
2222                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2223
2224                 /* We can reuse buffer as-is, just make sure it is local */
2225                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2226                         desc_cb->reuse_flag = 1;
2227                 else /* This page cannot be reused so discard it */
2228                         put_page(desc_cb->priv);
2229
2230                 ring_ptr_move_fw(ring, next_to_clean);
2231         } else {
2232                 u64_stats_update_begin(&ring->syncp);
2233                 ring->stats.seg_pkt_cnt++;
2234                 u64_stats_update_end(&ring->syncp);
2235
2236                 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2237
2238                 memcpy(__skb_put(skb, pull_len), va,
2239                        ALIGN(pull_len, sizeof(long)));
2240
2241                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2242                 ring_ptr_move_fw(ring, next_to_clean);
2243
2244                 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2245                         desc = &ring->desc[ring->next_to_clean];
2246                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2247                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2248                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2249                         ring_ptr_move_fw(ring, next_to_clean);
2250                         bnum++;
2251                 }
2252         }
2253
2254         *out_bnum = bnum;
2255
2256         l234info = le32_to_cpu(desc->rx.l234_info);
2257
2258         /* Based on hw strategy, the tag offloaded will be stored at
2259          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2260          * in one layer tag case.
2261          */
2262         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2263                 u16 vlan_tag;
2264
2265                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2266                         __vlan_hwaccel_put_tag(skb,
2267                                                htons(ETH_P_8021Q),
2268                                                vlan_tag);
2269         }
2270
2271         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2272                 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2273                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
2274                 u64_stats_update_begin(&ring->syncp);
2275                 ring->stats.non_vld_descs++;
2276                 u64_stats_update_end(&ring->syncp);
2277
2278                 dev_kfree_skb_any(skb);
2279                 return -EINVAL;
2280         }
2281
2282         if (unlikely((!desc->rx.pkt_len) ||
2283                      hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2284                 netdev_err(netdev, "truncated pkt\n");
2285                 u64_stats_update_begin(&ring->syncp);
2286                 ring->stats.err_pkt_len++;
2287                 u64_stats_update_end(&ring->syncp);
2288
2289                 dev_kfree_skb_any(skb);
2290                 return -EFAULT;
2291         }
2292
2293         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2294                 netdev_err(netdev, "L2 error pkt\n");
2295                 u64_stats_update_begin(&ring->syncp);
2296                 ring->stats.l2_err++;
2297                 u64_stats_update_end(&ring->syncp);
2298
2299                 dev_kfree_skb_any(skb);
2300                 return -EFAULT;
2301         }
2302
2303         u64_stats_update_begin(&ring->syncp);
2304         ring->stats.rx_pkts++;
2305         ring->stats.rx_bytes += skb->len;
2306         u64_stats_update_end(&ring->syncp);
2307
2308         ring->tqp_vector->rx_group.total_bytes += skb->len;
2309
2310         hns3_rx_checksum(ring, skb, desc);
2311         return 0;
2312 }
2313
2314 int hns3_clean_rx_ring(
2315                 struct hns3_enet_ring *ring, int budget,
2316                 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2317 {
2318 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2319         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2320         int recv_pkts, recv_bds, clean_count, err;
2321         int unused_count = hns3_desc_unused(ring);
2322         struct sk_buff *skb = NULL;
2323         int num, bnum = 0;
2324
2325         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2326         rmb(); /* Make sure num taken effect before the other data is touched */
2327
2328         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2329         num -= unused_count;
2330
2331         while (recv_pkts < budget && recv_bds < num) {
2332                 /* Reuse or realloc buffers */
2333                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2334                         hns3_nic_alloc_rx_buffers(ring,
2335                                                   clean_count + unused_count);
2336                         clean_count = 0;
2337                         unused_count = hns3_desc_unused(ring);
2338                 }
2339
2340                 /* Poll one pkt */
2341                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2342                 if (unlikely(!skb)) /* This fault cannot be repaired */
2343                         goto out;
2344
2345                 recv_bds += bnum;
2346                 clean_count += bnum;
2347                 if (unlikely(err)) {  /* Do jump the err */
2348                         recv_pkts++;
2349                         continue;
2350                 }
2351
2352                 /* Do update ip stack process */
2353                 skb->protocol = eth_type_trans(skb, netdev);
2354                 rx_fn(ring, skb);
2355
2356                 recv_pkts++;
2357         }
2358
2359 out:
2360         /* Make all data has been write before submit */
2361         if (clean_count + unused_count > 0)
2362                 hns3_nic_alloc_rx_buffers(ring,
2363                                           clean_count + unused_count);
2364
2365         return recv_pkts;
2366 }
2367
2368 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2369 {
2370         struct hns3_enet_tqp_vector *tqp_vector =
2371                                         ring_group->ring->tqp_vector;
2372         enum hns3_flow_level_range new_flow_level;
2373         int packets_per_msecs;
2374         int bytes_per_msecs;
2375         u32 time_passed_ms;
2376         u16 new_int_gl;
2377
2378         if (!tqp_vector->last_jiffies)
2379                 return false;
2380
2381         if (ring_group->total_packets == 0) {
2382                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2383                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
2384                 return true;
2385         }
2386
2387         /* Simple throttlerate management
2388          * 0-10MB/s   lower     (50000 ints/s)
2389          * 10-20MB/s   middle    (20000 ints/s)
2390          * 20-1249MB/s high      (18000 ints/s)
2391          * > 40000pps  ultra     (8000 ints/s)
2392          */
2393         new_flow_level = ring_group->coal.flow_level;
2394         new_int_gl = ring_group->coal.int_gl;
2395         time_passed_ms =
2396                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2397
2398         if (!time_passed_ms)
2399                 return false;
2400
2401         do_div(ring_group->total_packets, time_passed_ms);
2402         packets_per_msecs = ring_group->total_packets;
2403
2404         do_div(ring_group->total_bytes, time_passed_ms);
2405         bytes_per_msecs = ring_group->total_bytes;
2406
2407 #define HNS3_RX_LOW_BYTE_RATE 10000
2408 #define HNS3_RX_MID_BYTE_RATE 20000
2409
2410         switch (new_flow_level) {
2411         case HNS3_FLOW_LOW:
2412                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2413                         new_flow_level = HNS3_FLOW_MID;
2414                 break;
2415         case HNS3_FLOW_MID:
2416                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2417                         new_flow_level = HNS3_FLOW_HIGH;
2418                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2419                         new_flow_level = HNS3_FLOW_LOW;
2420                 break;
2421         case HNS3_FLOW_HIGH:
2422         case HNS3_FLOW_ULTRA:
2423         default:
2424                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2425                         new_flow_level = HNS3_FLOW_MID;
2426                 break;
2427         }
2428
2429 #define HNS3_RX_ULTRA_PACKET_RATE 40
2430
2431         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2432             &tqp_vector->rx_group == ring_group)
2433                 new_flow_level = HNS3_FLOW_ULTRA;
2434
2435         switch (new_flow_level) {
2436         case HNS3_FLOW_LOW:
2437                 new_int_gl = HNS3_INT_GL_50K;
2438                 break;
2439         case HNS3_FLOW_MID:
2440                 new_int_gl = HNS3_INT_GL_20K;
2441                 break;
2442         case HNS3_FLOW_HIGH:
2443                 new_int_gl = HNS3_INT_GL_18K;
2444                 break;
2445         case HNS3_FLOW_ULTRA:
2446                 new_int_gl = HNS3_INT_GL_8K;
2447                 break;
2448         default:
2449                 break;
2450         }
2451
2452         ring_group->total_bytes = 0;
2453         ring_group->total_packets = 0;
2454         ring_group->coal.flow_level = new_flow_level;
2455         if (new_int_gl != ring_group->coal.int_gl) {
2456                 ring_group->coal.int_gl = new_int_gl;
2457                 return true;
2458         }
2459         return false;
2460 }
2461
2462 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2463 {
2464         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2465         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2466         bool rx_update, tx_update;
2467
2468         if (tqp_vector->int_adapt_down > 0) {
2469                 tqp_vector->int_adapt_down--;
2470                 return;
2471         }
2472
2473         if (rx_group->coal.gl_adapt_enable) {
2474                 rx_update = hns3_get_new_int_gl(rx_group);
2475                 if (rx_update)
2476                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
2477                                                        rx_group->coal.int_gl);
2478         }
2479
2480         if (tx_group->coal.gl_adapt_enable) {
2481                 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2482                 if (tx_update)
2483                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
2484                                                        tx_group->coal.int_gl);
2485         }
2486
2487         tqp_vector->last_jiffies = jiffies;
2488         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
2489 }
2490
2491 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2492 {
2493         struct hns3_enet_ring *ring;
2494         int rx_pkt_total = 0;
2495
2496         struct hns3_enet_tqp_vector *tqp_vector =
2497                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2498         bool clean_complete = true;
2499         int rx_budget;
2500
2501         /* Since the actual Tx work is minimal, we can give the Tx a larger
2502          * budget and be more aggressive about cleaning up the Tx descriptors.
2503          */
2504         hns3_for_each_ring(ring, tqp_vector->tx_group) {
2505                 if (!hns3_clean_tx_ring(ring, budget))
2506                         clean_complete = false;
2507         }
2508
2509         /* make sure rx ring budget not smaller than 1 */
2510         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2511
2512         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2513                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2514                                                     hns3_rx_skb);
2515
2516                 if (rx_cleaned >= rx_budget)
2517                         clean_complete = false;
2518
2519                 rx_pkt_total += rx_cleaned;
2520         }
2521
2522         tqp_vector->rx_group.total_packets += rx_pkt_total;
2523
2524         if (!clean_complete)
2525                 return budget;
2526
2527         napi_complete(napi);
2528         hns3_update_new_int_gl(tqp_vector);
2529         hns3_mask_vector_irq(tqp_vector, 1);
2530
2531         return rx_pkt_total;
2532 }
2533
2534 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2535                                       struct hnae3_ring_chain_node *head)
2536 {
2537         struct pci_dev *pdev = tqp_vector->handle->pdev;
2538         struct hnae3_ring_chain_node *cur_chain = head;
2539         struct hnae3_ring_chain_node *chain;
2540         struct hns3_enet_ring *tx_ring;
2541         struct hns3_enet_ring *rx_ring;
2542
2543         tx_ring = tqp_vector->tx_group.ring;
2544         if (tx_ring) {
2545                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2546                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2547                               HNAE3_RING_TYPE_TX);
2548                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2549                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2550
2551                 cur_chain->next = NULL;
2552
2553                 while (tx_ring->next) {
2554                         tx_ring = tx_ring->next;
2555
2556                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2557                                              GFP_KERNEL);
2558                         if (!chain)
2559                                 goto err_free_chain;
2560
2561                         cur_chain->next = chain;
2562                         chain->tqp_index = tx_ring->tqp->tqp_index;
2563                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2564                                       HNAE3_RING_TYPE_TX);
2565                         hnae3_set_field(chain->int_gl_idx,
2566                                         HNAE3_RING_GL_IDX_M,
2567                                         HNAE3_RING_GL_IDX_S,
2568                                         HNAE3_RING_GL_TX);
2569
2570                         cur_chain = chain;
2571                 }
2572         }
2573
2574         rx_ring = tqp_vector->rx_group.ring;
2575         if (!tx_ring && rx_ring) {
2576                 cur_chain->next = NULL;
2577                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2578                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2579                               HNAE3_RING_TYPE_RX);
2580                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2581                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2582
2583                 rx_ring = rx_ring->next;
2584         }
2585
2586         while (rx_ring) {
2587                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2588                 if (!chain)
2589                         goto err_free_chain;
2590
2591                 cur_chain->next = chain;
2592                 chain->tqp_index = rx_ring->tqp->tqp_index;
2593                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2594                               HNAE3_RING_TYPE_RX);
2595                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2596                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2597
2598                 cur_chain = chain;
2599
2600                 rx_ring = rx_ring->next;
2601         }
2602
2603         return 0;
2604
2605 err_free_chain:
2606         cur_chain = head->next;
2607         while (cur_chain) {
2608                 chain = cur_chain->next;
2609                 devm_kfree(&pdev->dev, cur_chain);
2610                 cur_chain = chain;
2611         }
2612         head->next = NULL;
2613
2614         return -ENOMEM;
2615 }
2616
2617 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2618                                         struct hnae3_ring_chain_node *head)
2619 {
2620         struct pci_dev *pdev = tqp_vector->handle->pdev;
2621         struct hnae3_ring_chain_node *chain_tmp, *chain;
2622
2623         chain = head->next;
2624
2625         while (chain) {
2626                 chain_tmp = chain->next;
2627                 devm_kfree(&pdev->dev, chain);
2628                 chain = chain_tmp;
2629         }
2630 }
2631
2632 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2633                                    struct hns3_enet_ring *ring)
2634 {
2635         ring->next = group->ring;
2636         group->ring = ring;
2637
2638         group->count++;
2639 }
2640
2641 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2642 {
2643         struct hnae3_handle *h = priv->ae_handle;
2644         struct hns3_enet_tqp_vector *tqp_vector;
2645         int ret = 0;
2646         int i;
2647
2648         for (i = 0; i < priv->vector_num; i++) {
2649                 tqp_vector = &priv->tqp_vector[i];
2650                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2651                 tqp_vector->num_tqps = 0;
2652         }
2653
2654         for (i = 0; i < h->kinfo.num_tqps; i++) {
2655                 u16 vector_i = i % priv->vector_num;
2656                 u16 tqp_num = h->kinfo.num_tqps;
2657
2658                 tqp_vector = &priv->tqp_vector[vector_i];
2659
2660                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2661                                        priv->ring_data[i].ring);
2662
2663                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2664                                        priv->ring_data[i + tqp_num].ring);
2665
2666                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2667                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2668                 tqp_vector->num_tqps++;
2669         }
2670
2671         for (i = 0; i < priv->vector_num; i++) {
2672                 struct hnae3_ring_chain_node vector_ring_chain;
2673
2674                 tqp_vector = &priv->tqp_vector[i];
2675
2676                 tqp_vector->rx_group.total_bytes = 0;
2677                 tqp_vector->rx_group.total_packets = 0;
2678                 tqp_vector->tx_group.total_bytes = 0;
2679                 tqp_vector->tx_group.total_packets = 0;
2680                 tqp_vector->handle = h;
2681
2682                 ret = hns3_get_vector_ring_chain(tqp_vector,
2683                                                  &vector_ring_chain);
2684                 if (ret)
2685                         goto map_ring_fail;
2686
2687                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2688                         tqp_vector->vector_irq, &vector_ring_chain);
2689
2690                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2691
2692                 if (ret)
2693                         goto map_ring_fail;
2694
2695                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2696                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2697         }
2698
2699         return 0;
2700
2701 map_ring_fail:
2702         while (i--)
2703                 netif_napi_del(&priv->tqp_vector[i].napi);
2704
2705         return ret;
2706 }
2707
2708 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2709 {
2710 #define HNS3_VECTOR_PF_MAX_NUM          64
2711
2712         struct hnae3_handle *h = priv->ae_handle;
2713         struct hns3_enet_tqp_vector *tqp_vector;
2714         struct hnae3_vector_info *vector;
2715         struct pci_dev *pdev = h->pdev;
2716         u16 tqp_num = h->kinfo.num_tqps;
2717         u16 vector_num;
2718         int ret = 0;
2719         u16 i;
2720
2721         /* RSS size, cpu online and vector_num should be the same */
2722         /* Should consider 2p/4p later */
2723         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2724         vector_num = min_t(u16, vector_num, HNS3_VECTOR_PF_MAX_NUM);
2725
2726         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2727                               GFP_KERNEL);
2728         if (!vector)
2729                 return -ENOMEM;
2730
2731         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2732
2733         priv->vector_num = vector_num;
2734         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2735                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2736                              GFP_KERNEL);
2737         if (!priv->tqp_vector) {
2738                 ret = -ENOMEM;
2739                 goto out;
2740         }
2741
2742         for (i = 0; i < priv->vector_num; i++) {
2743                 tqp_vector = &priv->tqp_vector[i];
2744                 tqp_vector->idx = i;
2745                 tqp_vector->mask_addr = vector[i].io_addr;
2746                 tqp_vector->vector_irq = vector[i].vector;
2747                 hns3_vector_gl_rl_init(tqp_vector, priv);
2748         }
2749
2750 out:
2751         devm_kfree(&pdev->dev, vector);
2752         return ret;
2753 }
2754
2755 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2756 {
2757         group->ring = NULL;
2758         group->count = 0;
2759 }
2760
2761 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2762 {
2763         struct hnae3_ring_chain_node vector_ring_chain;
2764         struct hnae3_handle *h = priv->ae_handle;
2765         struct hns3_enet_tqp_vector *tqp_vector;
2766         int i, ret;
2767
2768         for (i = 0; i < priv->vector_num; i++) {
2769                 tqp_vector = &priv->tqp_vector[i];
2770
2771                 ret = hns3_get_vector_ring_chain(tqp_vector,
2772                                                  &vector_ring_chain);
2773                 if (ret)
2774                         return ret;
2775
2776                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2777                         tqp_vector->vector_irq, &vector_ring_chain);
2778                 if (ret)
2779                         return ret;
2780
2781                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2782
2783                 if (tqp_vector->irq_init_flag == HNS3_VECTOR_INITED) {
2784                         irq_set_affinity_notifier(tqp_vector->vector_irq,
2785                                                   NULL);
2786                         irq_set_affinity_hint(tqp_vector->vector_irq, NULL);
2787                         free_irq(tqp_vector->vector_irq, tqp_vector);
2788                         tqp_vector->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2789                 }
2790
2791                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2792                 hns3_clear_ring_group(&tqp_vector->rx_group);
2793                 hns3_clear_ring_group(&tqp_vector->tx_group);
2794                 netif_napi_del(&priv->tqp_vector[i].napi);
2795         }
2796
2797         return 0;
2798 }
2799
2800 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2801 {
2802         struct hnae3_handle *h = priv->ae_handle;
2803         struct pci_dev *pdev = h->pdev;
2804         int i, ret;
2805
2806         for (i = 0; i < priv->vector_num; i++) {
2807                 struct hns3_enet_tqp_vector *tqp_vector;
2808
2809                 tqp_vector = &priv->tqp_vector[i];
2810                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2811                 if (ret)
2812                         return ret;
2813         }
2814
2815         devm_kfree(&pdev->dev, priv->tqp_vector);
2816         return 0;
2817 }
2818
2819 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2820                              int ring_type)
2821 {
2822         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2823         int queue_num = priv->ae_handle->kinfo.num_tqps;
2824         struct pci_dev *pdev = priv->ae_handle->pdev;
2825         struct hns3_enet_ring *ring;
2826
2827         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2828         if (!ring)
2829                 return -ENOMEM;
2830
2831         if (ring_type == HNAE3_RING_TYPE_TX) {
2832                 ring_data[q->tqp_index].ring = ring;
2833                 ring_data[q->tqp_index].queue_index = q->tqp_index;
2834                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2835         } else {
2836                 ring_data[q->tqp_index + queue_num].ring = ring;
2837                 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
2838                 ring->io_base = q->io_base;
2839         }
2840
2841         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2842
2843         ring->tqp = q;
2844         ring->desc = NULL;
2845         ring->desc_cb = NULL;
2846         ring->dev = priv->dev;
2847         ring->desc_dma_addr = 0;
2848         ring->buf_size = q->buf_size;
2849         ring->desc_num = q->desc_num;
2850         ring->next_to_use = 0;
2851         ring->next_to_clean = 0;
2852
2853         return 0;
2854 }
2855
2856 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2857                               struct hns3_nic_priv *priv)
2858 {
2859         int ret;
2860
2861         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2862         if (ret)
2863                 return ret;
2864
2865         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2866         if (ret) {
2867                 devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
2868                 return ret;
2869         }
2870
2871         return 0;
2872 }
2873
2874 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2875 {
2876         struct hnae3_handle *h = priv->ae_handle;
2877         struct pci_dev *pdev = h->pdev;
2878         int i, ret;
2879
2880         priv->ring_data =  devm_kzalloc(&pdev->dev,
2881                                         array3_size(h->kinfo.num_tqps,
2882                                                     sizeof(*priv->ring_data),
2883                                                     2),
2884                                         GFP_KERNEL);
2885         if (!priv->ring_data)
2886                 return -ENOMEM;
2887
2888         for (i = 0; i < h->kinfo.num_tqps; i++) {
2889                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2890                 if (ret)
2891                         goto err;
2892         }
2893
2894         return 0;
2895 err:
2896         while (i--) {
2897                 devm_kfree(priv->dev, priv->ring_data[i].ring);
2898                 devm_kfree(priv->dev,
2899                            priv->ring_data[i + h->kinfo.num_tqps].ring);
2900         }
2901
2902         devm_kfree(&pdev->dev, priv->ring_data);
2903         return ret;
2904 }
2905
2906 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2907 {
2908         struct hnae3_handle *h = priv->ae_handle;
2909         int i;
2910
2911         for (i = 0; i < h->kinfo.num_tqps; i++) {
2912                 devm_kfree(priv->dev, priv->ring_data[i].ring);
2913                 devm_kfree(priv->dev,
2914                            priv->ring_data[i + h->kinfo.num_tqps].ring);
2915         }
2916         devm_kfree(priv->dev, priv->ring_data);
2917 }
2918
2919 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2920 {
2921         int ret;
2922
2923         if (ring->desc_num <= 0 || ring->buf_size <= 0)
2924                 return -EINVAL;
2925
2926         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2927                                 GFP_KERNEL);
2928         if (!ring->desc_cb) {
2929                 ret = -ENOMEM;
2930                 goto out;
2931         }
2932
2933         ret = hns3_alloc_desc(ring);
2934         if (ret)
2935                 goto out_with_desc_cb;
2936
2937         if (!HNAE3_IS_TX_RING(ring)) {
2938                 ret = hns3_alloc_ring_buffers(ring);
2939                 if (ret)
2940                         goto out_with_desc;
2941         }
2942
2943         return 0;
2944
2945 out_with_desc:
2946         hns3_free_desc(ring);
2947 out_with_desc_cb:
2948         kfree(ring->desc_cb);
2949         ring->desc_cb = NULL;
2950 out:
2951         return ret;
2952 }
2953
2954 static void hns3_fini_ring(struct hns3_enet_ring *ring)
2955 {
2956         hns3_free_desc(ring);
2957         kfree(ring->desc_cb);
2958         ring->desc_cb = NULL;
2959         ring->next_to_clean = 0;
2960         ring->next_to_use = 0;
2961 }
2962
2963 static int hns3_buf_size2type(u32 buf_size)
2964 {
2965         int bd_size_type;
2966
2967         switch (buf_size) {
2968         case 512:
2969                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2970                 break;
2971         case 1024:
2972                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2973                 break;
2974         case 2048:
2975                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2976                 break;
2977         case 4096:
2978                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2979                 break;
2980         default:
2981                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2982         }
2983
2984         return bd_size_type;
2985 }
2986
2987 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2988 {
2989         dma_addr_t dma = ring->desc_dma_addr;
2990         struct hnae3_queue *q = ring->tqp;
2991
2992         if (!HNAE3_IS_TX_RING(ring)) {
2993                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2994                                (u32)dma);
2995                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2996                                (u32)((dma >> 31) >> 1));
2997
2998                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2999                                hns3_buf_size2type(ring->buf_size));
3000                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3001                                ring->desc_num / 8 - 1);
3002
3003         } else {
3004                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3005                                (u32)dma);
3006                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3007                                (u32)((dma >> 31) >> 1));
3008
3009                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3010                                ring->desc_num / 8 - 1);
3011         }
3012 }
3013
3014 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3015 {
3016         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3017         int i;
3018
3019         for (i = 0; i < HNAE3_MAX_TC; i++) {
3020                 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3021                 int j;
3022
3023                 if (!tc_info->enable)
3024                         continue;
3025
3026                 for (j = 0; j < tc_info->tqp_count; j++) {
3027                         struct hnae3_queue *q;
3028
3029                         q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3030                         hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3031                                        tc_info->tc);
3032                 }
3033         }
3034 }
3035
3036 int hns3_init_all_ring(struct hns3_nic_priv *priv)
3037 {
3038         struct hnae3_handle *h = priv->ae_handle;
3039         int ring_num = h->kinfo.num_tqps * 2;
3040         int i, j;
3041         int ret;
3042
3043         for (i = 0; i < ring_num; i++) {
3044                 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3045                 if (ret) {
3046                         dev_err(priv->dev,
3047                                 "Alloc ring memory fail! ret=%d\n", ret);
3048                         goto out_when_alloc_ring_memory;
3049                 }
3050
3051                 u64_stats_init(&priv->ring_data[i].ring->syncp);
3052         }
3053
3054         return 0;
3055
3056 out_when_alloc_ring_memory:
3057         for (j = i - 1; j >= 0; j--)
3058                 hns3_fini_ring(priv->ring_data[j].ring);
3059
3060         return -ENOMEM;
3061 }
3062
3063 int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
3064 {
3065         struct hnae3_handle *h = priv->ae_handle;
3066         int i;
3067
3068         for (i = 0; i < h->kinfo.num_tqps; i++) {
3069                 if (h->ae_algo->ops->reset_queue)
3070                         h->ae_algo->ops->reset_queue(h, i);
3071
3072                 hns3_fini_ring(priv->ring_data[i].ring);
3073                 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3074         }
3075         return 0;
3076 }
3077
3078 /* Set mac addr if it is configured. or leave it to the AE driver */
3079 static void hns3_init_mac_addr(struct net_device *netdev, bool init)
3080 {
3081         struct hns3_nic_priv *priv = netdev_priv(netdev);
3082         struct hnae3_handle *h = priv->ae_handle;
3083         u8 mac_addr_temp[ETH_ALEN];
3084
3085         if (h->ae_algo->ops->get_mac_addr && init) {
3086                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3087                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3088         }
3089
3090         /* Check if the MAC address is valid, if not get a random one */
3091         if (!is_valid_ether_addr(netdev->dev_addr)) {
3092                 eth_hw_addr_random(netdev);
3093                 dev_warn(priv->dev, "using random MAC address %pM\n",
3094                          netdev->dev_addr);
3095         }
3096
3097         if (h->ae_algo->ops->set_mac_addr)
3098                 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
3099
3100 }
3101
3102 static void hns3_uninit_mac_addr(struct net_device *netdev)
3103 {
3104         struct hns3_nic_priv *priv = netdev_priv(netdev);
3105         struct hnae3_handle *h = priv->ae_handle;
3106
3107         if (h->ae_algo->ops->rm_uc_addr)
3108                 h->ae_algo->ops->rm_uc_addr(h, netdev->dev_addr);
3109 }
3110
3111 static void hns3_nic_set_priv_ops(struct net_device *netdev)
3112 {
3113         struct hns3_nic_priv *priv = netdev_priv(netdev);
3114
3115         if ((netdev->features & NETIF_F_TSO) ||
3116             (netdev->features & NETIF_F_TSO6)) {
3117                 priv->ops.fill_desc = hns3_fill_desc_tso;
3118                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3119         } else {
3120                 priv->ops.fill_desc = hns3_fill_desc;
3121                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3122         }
3123 }
3124
3125 static int hns3_client_init(struct hnae3_handle *handle)
3126 {
3127         struct pci_dev *pdev = handle->pdev;
3128         struct hns3_nic_priv *priv;
3129         struct net_device *netdev;
3130         int ret;
3131
3132         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
3133                                    hns3_get_max_available_channels(handle));
3134         if (!netdev)
3135                 return -ENOMEM;
3136
3137         priv = netdev_priv(netdev);
3138         priv->dev = &pdev->dev;
3139         priv->netdev = netdev;
3140         priv->ae_handle = handle;
3141         priv->ae_handle->last_reset_time = jiffies;
3142         priv->tx_timeout_count = 0;
3143
3144         handle->kinfo.netdev = netdev;
3145         handle->priv = (void *)priv;
3146
3147         hns3_init_mac_addr(netdev, true);
3148
3149         hns3_set_default_feature(netdev);
3150
3151         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3152         netdev->priv_flags |= IFF_UNICAST_FLT;
3153         netdev->netdev_ops = &hns3_nic_netdev_ops;
3154         SET_NETDEV_DEV(netdev, &pdev->dev);
3155         hns3_ethtool_set_ops(netdev);
3156         hns3_nic_set_priv_ops(netdev);
3157
3158         /* Carrier off reporting is important to ethtool even BEFORE open */
3159         netif_carrier_off(netdev);
3160
3161         if (handle->flags & HNAE3_SUPPORT_VF)
3162                 handle->reset_level = HNAE3_VF_RESET;
3163         else
3164                 handle->reset_level = HNAE3_FUNC_RESET;
3165
3166         ret = hns3_get_ring_config(priv);
3167         if (ret) {
3168                 ret = -ENOMEM;
3169                 goto out_get_ring_cfg;
3170         }
3171
3172         ret = hns3_nic_alloc_vector_data(priv);
3173         if (ret) {
3174                 ret = -ENOMEM;
3175                 goto out_alloc_vector_data;
3176         }
3177
3178         ret = hns3_nic_init_vector_data(priv);
3179         if (ret) {
3180                 ret = -ENOMEM;
3181                 goto out_init_vector_data;
3182         }
3183
3184         ret = hns3_init_all_ring(priv);
3185         if (ret) {
3186                 ret = -ENOMEM;
3187                 goto out_init_ring_data;
3188         }
3189
3190         ret = register_netdev(netdev);
3191         if (ret) {
3192                 dev_err(priv->dev, "probe register netdev fail!\n");
3193                 goto out_reg_netdev_fail;
3194         }
3195
3196         hns3_dcbnl_setup(handle);
3197
3198         /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3199         netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3200
3201         return ret;
3202
3203 out_reg_netdev_fail:
3204 out_init_ring_data:
3205         (void)hns3_nic_uninit_vector_data(priv);
3206 out_init_vector_data:
3207         hns3_nic_dealloc_vector_data(priv);
3208 out_alloc_vector_data:
3209         priv->ring_data = NULL;
3210 out_get_ring_cfg:
3211         priv->ae_handle = NULL;
3212         free_netdev(netdev);
3213         return ret;
3214 }
3215
3216 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3217 {
3218         struct net_device *netdev = handle->kinfo.netdev;
3219         struct hns3_nic_priv *priv = netdev_priv(netdev);
3220         int ret;
3221
3222         if (netdev->reg_state != NETREG_UNINITIALIZED)
3223                 unregister_netdev(netdev);
3224
3225         hns3_force_clear_all_rx_ring(handle);
3226
3227         ret = hns3_nic_uninit_vector_data(priv);
3228         if (ret)
3229                 netdev_err(netdev, "uninit vector error\n");
3230
3231         ret = hns3_nic_dealloc_vector_data(priv);
3232         if (ret)
3233                 netdev_err(netdev, "dealloc vector error\n");
3234
3235         ret = hns3_uninit_all_ring(priv);
3236         if (ret)
3237                 netdev_err(netdev, "uninit ring error\n");
3238
3239         hns3_put_ring_config(priv);
3240
3241         priv->ring_data = NULL;
3242
3243         hns3_uninit_mac_addr(netdev);
3244
3245         free_netdev(netdev);
3246 }
3247
3248 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3249 {
3250         struct net_device *netdev = handle->kinfo.netdev;
3251
3252         if (!netdev)
3253                 return;
3254
3255         if (linkup) {
3256                 netif_carrier_on(netdev);
3257                 netif_tx_wake_all_queues(netdev);
3258                 netdev_info(netdev, "link up\n");
3259         } else {
3260                 netif_carrier_off(netdev);
3261                 netif_tx_stop_all_queues(netdev);
3262                 netdev_info(netdev, "link down\n");
3263         }
3264 }
3265
3266 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3267 {
3268         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3269         struct net_device *ndev = kinfo->netdev;
3270         bool if_running;
3271         int ret;
3272
3273         if (tc > HNAE3_MAX_TC)
3274                 return -EINVAL;
3275
3276         if (!ndev)
3277                 return -ENODEV;
3278
3279         if_running = netif_running(ndev);
3280
3281         if (if_running) {
3282                 (void)hns3_nic_net_stop(ndev);
3283                 msleep(100);
3284         }
3285
3286         ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3287                 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3288         if (ret)
3289                 goto err_out;
3290
3291         ret = hns3_nic_set_real_num_queue(ndev);
3292
3293 err_out:
3294         if (if_running)
3295                 (void)hns3_nic_net_open(ndev);
3296
3297         return ret;
3298 }
3299
3300 static void hns3_recover_hw_addr(struct net_device *ndev)
3301 {
3302         struct netdev_hw_addr_list *list;
3303         struct netdev_hw_addr *ha, *tmp;
3304
3305         /* go through and sync uc_addr entries to the device */
3306         list = &ndev->uc;
3307         list_for_each_entry_safe(ha, tmp, &list->list, list)
3308                 hns3_nic_uc_sync(ndev, ha->addr);
3309
3310         /* go through and sync mc_addr entries to the device */
3311         list = &ndev->mc;
3312         list_for_each_entry_safe(ha, tmp, &list->list, list)
3313                 hns3_nic_mc_sync(ndev, ha->addr);
3314 }
3315
3316 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
3317 {
3318         while (ring->next_to_clean != ring->next_to_use) {
3319                 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
3320                 hns3_free_buffer_detach(ring, ring->next_to_clean);
3321                 ring_ptr_move_fw(ring, next_to_clean);
3322         }
3323 }
3324
3325 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3326 {
3327         struct hns3_desc_cb res_cbs;
3328         int ret;
3329
3330         while (ring->next_to_use != ring->next_to_clean) {
3331                 /* When a buffer is not reused, it's memory has been
3332                  * freed in hns3_handle_rx_bd or will be freed by
3333                  * stack, so we need to replace the buffer here.
3334                  */
3335                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3336                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
3337                         if (ret) {
3338                                 u64_stats_update_begin(&ring->syncp);
3339                                 ring->stats.sw_err_cnt++;
3340                                 u64_stats_update_end(&ring->syncp);
3341                                 /* if alloc new buffer fail, exit directly
3342                                  * and reclear in up flow.
3343                                  */
3344                                 netdev_warn(ring->tqp->handle->kinfo.netdev,
3345                                             "reserve buffer map failed, ret = %d\n",
3346                                             ret);
3347                                 return ret;
3348                         }
3349                         hns3_replace_buffer(ring, ring->next_to_use,
3350                                             &res_cbs);
3351                 }
3352                 ring_ptr_move_fw(ring, next_to_use);
3353         }
3354
3355         return 0;
3356 }
3357
3358 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
3359 {
3360         while (ring->next_to_use != ring->next_to_clean) {
3361                 /* When a buffer is not reused, it's memory has been
3362                  * freed in hns3_handle_rx_bd or will be freed by
3363                  * stack, so only need to unmap the buffer here.
3364                  */
3365                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3366                         hns3_unmap_buffer(ring,
3367                                           &ring->desc_cb[ring->next_to_use]);
3368                         ring->desc_cb[ring->next_to_use].dma = 0;
3369                 }
3370
3371                 ring_ptr_move_fw(ring, next_to_use);
3372         }
3373 }
3374
3375 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3376 {
3377         struct net_device *ndev = h->kinfo.netdev;
3378         struct hns3_nic_priv *priv = netdev_priv(ndev);
3379         struct hns3_enet_ring *ring;
3380         u32 i;
3381
3382         for (i = 0; i < h->kinfo.num_tqps; i++) {
3383                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3384                 hns3_force_clear_rx_ring(ring);
3385         }
3386 }
3387
3388 static void hns3_clear_all_ring(struct hnae3_handle *h)
3389 {
3390         struct net_device *ndev = h->kinfo.netdev;
3391         struct hns3_nic_priv *priv = netdev_priv(ndev);
3392         u32 i;
3393
3394         for (i = 0; i < h->kinfo.num_tqps; i++) {
3395                 struct netdev_queue *dev_queue;
3396                 struct hns3_enet_ring *ring;
3397
3398                 ring = priv->ring_data[i].ring;
3399                 hns3_clear_tx_ring(ring);
3400                 dev_queue = netdev_get_tx_queue(ndev,
3401                                                 priv->ring_data[i].queue_index);
3402                 netdev_tx_reset_queue(dev_queue);
3403
3404                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3405                 /* Continue to clear other rings even if clearing some
3406                  * rings failed.
3407                  */
3408                 hns3_clear_rx_ring(ring);
3409         }
3410 }
3411
3412 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3413 {
3414         struct net_device *ndev = h->kinfo.netdev;
3415         struct hns3_nic_priv *priv = netdev_priv(ndev);
3416         struct hns3_enet_ring *rx_ring;
3417         int i, j;
3418         int ret;
3419
3420         for (i = 0; i < h->kinfo.num_tqps; i++) {
3421                 h->ae_algo->ops->reset_queue(h, i);
3422                 hns3_init_ring_hw(priv->ring_data[i].ring);
3423
3424                 /* We need to clear tx ring here because self test will
3425                  * use the ring and will not run down before up
3426                  */
3427                 hns3_clear_tx_ring(priv->ring_data[i].ring);
3428                 priv->ring_data[i].ring->next_to_clean = 0;
3429                 priv->ring_data[i].ring->next_to_use = 0;
3430
3431                 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3432                 hns3_init_ring_hw(rx_ring);
3433                 ret = hns3_clear_rx_ring(rx_ring);
3434                 if (ret)
3435                         return ret;
3436
3437                 /* We can not know the hardware head and tail when this
3438                  * function is called in reset flow, so we reuse all desc.
3439                  */
3440                 for (j = 0; j < rx_ring->desc_num; j++)
3441                         hns3_reuse_buffer(rx_ring, j);
3442
3443                 rx_ring->next_to_clean = 0;
3444                 rx_ring->next_to_use = 0;
3445         }
3446
3447         hns3_init_tx_ring_tc(priv);
3448
3449         return 0;
3450 }
3451
3452 static void hns3_store_coal(struct hns3_nic_priv *priv)
3453 {
3454         /* ethtool only support setting and querying one coal
3455          * configuation for now, so save the vector 0' coal
3456          * configuation here in order to restore it.
3457          */
3458         memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3459                sizeof(struct hns3_enet_coalesce));
3460         memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3461                sizeof(struct hns3_enet_coalesce));
3462 }
3463
3464 static void hns3_restore_coal(struct hns3_nic_priv *priv)
3465 {
3466         u16 vector_num = priv->vector_num;
3467         int i;
3468
3469         for (i = 0; i < vector_num; i++) {
3470                 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3471                        sizeof(struct hns3_enet_coalesce));
3472                 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3473                        sizeof(struct hns3_enet_coalesce));
3474         }
3475 }
3476
3477 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3478 {
3479         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3480         struct net_device *ndev = kinfo->netdev;
3481
3482         if (!netif_running(ndev))
3483                 return 0;
3484
3485         return hns3_nic_net_stop(ndev);
3486 }
3487
3488 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3489 {
3490         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3491         int ret = 0;
3492
3493         if (netif_running(kinfo->netdev)) {
3494                 ret = hns3_nic_net_up(kinfo->netdev);
3495                 if (ret) {
3496                         netdev_err(kinfo->netdev,
3497                                    "hns net up fail, ret=%d!\n", ret);
3498                         return ret;
3499                 }
3500                 handle->last_reset_time = jiffies;
3501         }
3502
3503         return ret;
3504 }
3505
3506 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3507 {
3508         struct net_device *netdev = handle->kinfo.netdev;
3509         struct hns3_nic_priv *priv = netdev_priv(netdev);
3510         int ret;
3511
3512         hns3_init_mac_addr(netdev, false);
3513         hns3_nic_set_rx_mode(netdev);
3514         hns3_recover_hw_addr(netdev);
3515
3516         /* Hardware table is only clear when pf resets */
3517         if (!(handle->flags & HNAE3_SUPPORT_VF))
3518                 hns3_restore_vlan(netdev);
3519
3520         /* Carrier off reporting is important to ethtool even BEFORE open */
3521         netif_carrier_off(netdev);
3522
3523         hns3_restore_coal(priv);
3524
3525         ret = hns3_nic_init_vector_data(priv);
3526         if (ret)
3527                 return ret;
3528
3529         ret = hns3_init_all_ring(priv);
3530         if (ret) {
3531                 hns3_nic_uninit_vector_data(priv);
3532                 priv->ring_data = NULL;
3533         }
3534
3535         return ret;
3536 }
3537
3538 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3539 {
3540         struct net_device *netdev = handle->kinfo.netdev;
3541         struct hns3_nic_priv *priv = netdev_priv(netdev);
3542         int ret;
3543
3544         hns3_force_clear_all_rx_ring(handle);
3545
3546         ret = hns3_nic_uninit_vector_data(priv);
3547         if (ret) {
3548                 netdev_err(netdev, "uninit vector error\n");
3549                 return ret;
3550         }
3551
3552         hns3_store_coal(priv);
3553
3554         ret = hns3_uninit_all_ring(priv);
3555         if (ret)
3556                 netdev_err(netdev, "uninit ring error\n");
3557
3558         hns3_uninit_mac_addr(netdev);
3559
3560         return ret;
3561 }
3562
3563 static int hns3_reset_notify(struct hnae3_handle *handle,
3564                              enum hnae3_reset_notify_type type)
3565 {
3566         int ret = 0;
3567
3568         switch (type) {
3569         case HNAE3_UP_CLIENT:
3570                 ret = hns3_reset_notify_up_enet(handle);
3571                 break;
3572         case HNAE3_DOWN_CLIENT:
3573                 ret = hns3_reset_notify_down_enet(handle);
3574                 break;
3575         case HNAE3_INIT_CLIENT:
3576                 ret = hns3_reset_notify_init_enet(handle);
3577                 break;
3578         case HNAE3_UNINIT_CLIENT:
3579                 ret = hns3_reset_notify_uninit_enet(handle);
3580                 break;
3581         default:
3582                 break;
3583         }
3584
3585         return ret;
3586 }
3587
3588 static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
3589 {
3590         struct hns3_nic_priv *priv = netdev_priv(netdev);
3591         struct hnae3_handle *h = hns3_get_handle(netdev);
3592         int ret;
3593
3594         ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3595         if (ret)
3596                 return ret;
3597
3598         ret = hns3_get_ring_config(priv);
3599         if (ret)
3600                 return ret;
3601
3602         ret = hns3_nic_alloc_vector_data(priv);
3603         if (ret)
3604                 goto err_alloc_vector;
3605
3606         hns3_restore_coal(priv);
3607
3608         ret = hns3_nic_init_vector_data(priv);
3609         if (ret)
3610                 goto err_uninit_vector;
3611
3612         ret = hns3_init_all_ring(priv);
3613         if (ret)
3614                 goto err_put_ring;
3615
3616         return 0;
3617
3618 err_put_ring:
3619         hns3_put_ring_config(priv);
3620 err_uninit_vector:
3621         hns3_nic_uninit_vector_data(priv);
3622 err_alloc_vector:
3623         hns3_nic_dealloc_vector_data(priv);
3624         return ret;
3625 }
3626
3627 static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3628 {
3629         return (new_tqp_num / num_tc) * num_tc;
3630 }
3631
3632 int hns3_set_channels(struct net_device *netdev,
3633                       struct ethtool_channels *ch)
3634 {
3635         struct hns3_nic_priv *priv = netdev_priv(netdev);
3636         struct hnae3_handle *h = hns3_get_handle(netdev);
3637         struct hnae3_knic_private_info *kinfo = &h->kinfo;
3638         bool if_running = netif_running(netdev);
3639         u32 new_tqp_num = ch->combined_count;
3640         u16 org_tqp_num;
3641         int ret;
3642
3643         if (ch->rx_count || ch->tx_count)
3644                 return -EINVAL;
3645
3646         if (new_tqp_num > hns3_get_max_available_channels(h) ||
3647             new_tqp_num < kinfo->num_tc) {
3648                 dev_err(&netdev->dev,
3649                         "Change tqps fail, the tqp range is from %d to %d",
3650                         kinfo->num_tc,
3651                         hns3_get_max_available_channels(h));
3652                 return -EINVAL;
3653         }
3654
3655         new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3656         if (kinfo->num_tqps == new_tqp_num)
3657                 return 0;
3658
3659         if (if_running)
3660                 hns3_nic_net_stop(netdev);
3661
3662         ret = hns3_nic_uninit_vector_data(priv);
3663         if (ret) {
3664                 dev_err(&netdev->dev,
3665                         "Unbind vector with tqp fail, nothing is changed");
3666                 goto open_netdev;
3667         }
3668
3669         hns3_store_coal(priv);
3670
3671         hns3_nic_dealloc_vector_data(priv);
3672
3673         hns3_uninit_all_ring(priv);
3674         hns3_put_ring_config(priv);
3675
3676         org_tqp_num = h->kinfo.num_tqps;
3677         ret = hns3_modify_tqp_num(netdev, new_tqp_num);
3678         if (ret) {
3679                 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
3680                 if (ret) {
3681                         /* If revert to old tqp failed, fatal error occurred */
3682                         dev_err(&netdev->dev,
3683                                 "Revert to old tqp num fail, ret=%d", ret);
3684                         return ret;
3685                 }
3686                 dev_info(&netdev->dev,
3687                          "Change tqp num fail, Revert to old tqp num");
3688         }
3689
3690 open_netdev:
3691         if (if_running)
3692                 hns3_nic_net_open(netdev);
3693
3694         return ret;
3695 }
3696
3697 static const struct hnae3_client_ops client_ops = {
3698         .init_instance = hns3_client_init,
3699         .uninit_instance = hns3_client_uninit,
3700         .link_status_change = hns3_link_status_change,
3701         .setup_tc = hns3_client_setup_tc,
3702         .reset_notify = hns3_reset_notify,
3703 };
3704
3705 /* hns3_init_module - Driver registration routine
3706  * hns3_init_module is the first routine called when the driver is
3707  * loaded. All it does is register with the PCI subsystem.
3708  */
3709 static int __init hns3_init_module(void)
3710 {
3711         int ret;
3712
3713         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3714         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3715
3716         client.type = HNAE3_CLIENT_KNIC;
3717         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3718                  hns3_driver_name);
3719
3720         client.ops = &client_ops;
3721
3722         INIT_LIST_HEAD(&client.node);
3723
3724         ret = hnae3_register_client(&client);
3725         if (ret)
3726                 return ret;
3727
3728         ret = pci_register_driver(&hns3_driver);
3729         if (ret)
3730                 hnae3_unregister_client(&client);
3731
3732         return ret;
3733 }
3734 module_init(hns3_init_module);
3735
3736 /* hns3_exit_module - Driver exit cleanup routine
3737  * hns3_exit_module is called just before the driver is removed
3738  * from memory.
3739  */
3740 static void __exit hns3_exit_module(void)
3741 {
3742         pci_unregister_driver(&hns3_driver);
3743         hnae3_unregister_client(&client);
3744 }
3745 module_exit(hns3_exit_module);
3746
3747 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3748 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3749 MODULE_LICENSE("GPL");
3750 MODULE_ALIAS("pci:hns-nic");
3751 MODULE_VERSION(HNS3_MOD_VERSION);