drivers/net/virtio_net.c

   1 /* A network driver using virtio.
   2  *
   3  * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  17  */
  18 //#define DEBUG
  19 #include <linux/netdevice.h>
  20 #include <linux/etherdevice.h>
  21 #include <linux/ethtool.h>
  22 #include <linux/module.h>
  23 #include <linux/virtio.h>
  24 #include <linux/virtio_net.h>
  25 #include <linux/bpf.h>
  26 #include <linux/bpf_trace.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/if_vlan.h>
  29 #include <linux/slab.h>
  30 #include <linux/cpu.h>
  31 #include <linux/average.h>
  32 #include <net/route.h>
  33
  34 static int napi_weight = NAPI_POLL_WEIGHT;
  35 module_param(napi_weight, int, 0444);
  36
  37 static bool csum = true, gso = true, napi_tx;
  38 module_param(csum, bool, 0444);
  39 module_param(gso, bool, 0444);
  40 module_param(napi_tx, bool, 0644);
  41
  42 /* FIXME: MTU in config. */
  43 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  44 #define GOOD_COPY_LEN   128
  45
  46 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
  47
  48 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
  49 #define VIRTIO_XDP_HEADROOM 256
  50
  51 /* RX packet size EWMA. The average packet size is used to determine the packet
  52  * buffer size when refilling RX rings. As the entire RX ring may be refilled
  53  * at once, the weight is chosen so that the EWMA will be insensitive to short-
  54  * term, transient changes in packet size.
  55  */
  56 DECLARE_EWMA(pkt_len, 0, 64)
  57
  58 #define VIRTNET_DRIVER_VERSION "1.0.0"
  59
  60 static const unsigned long guest_offloads[] = {
  61         VIRTIO_NET_F_GUEST_TSO4,
  62         VIRTIO_NET_F_GUEST_TSO6,
  63         VIRTIO_NET_F_GUEST_ECN,
  64         VIRTIO_NET_F_GUEST_UFO,
  65         VIRTIO_NET_F_GUEST_CSUM
  66 };
  67
  68 struct virtnet_stats {
  69         struct u64_stats_sync tx_syncp;
  70         struct u64_stats_sync rx_syncp;
  71         u64 tx_bytes;
  72         u64 tx_packets;
  73
  74         u64 rx_bytes;
  75         u64 rx_packets;
  76 };
  77
  78 /* Internal representation of a send virtqueue */
  79 struct send_queue {
  80         /* Virtqueue associated with this send _queue */
  81         struct virtqueue *vq;
  82
  83         /* TX: fragments + linear part + virtio header */
  84         struct scatterlist sg[MAX_SKB_FRAGS + 2];
  85
  86         /* Name of the send queue: output.$index */
  87         char name[40];
  88
  89         struct napi_struct napi;
  90 };
  91
  92 /* Internal representation of a receive virtqueue */
  93 struct receive_queue {
  94         /* Virtqueue associated with this receive_queue */
  95         struct virtqueue *vq;
  96
  97         struct napi_struct napi;
  98
  99         struct bpf_prog __rcu *xdp_prog;
 100
 101         /* Chain pages by the private ptr. */
 102         struct page *pages;
 103
 104         /* Average packet length for mergeable receive buffers. */
 105         struct ewma_pkt_len mrg_avg_pkt_len;
 106
 107         /* Page frag for packet buffer allocation. */
 108         struct page_frag alloc_frag;
 109
 110         /* RX: fragments + linear part + virtio header */
 111         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 112
 113         /* Min single buffer size for mergeable buffers case. */
 114         unsigned int min_buf_len;
 115
 116         /* Name of this receive queue: input.$index */
 117         char name[40];
 118 };
 119
 120 /* Control VQ buffers: protected by the rtnl lock */
 121 struct control_buf {
 122         struct virtio_net_ctrl_hdr hdr;
 123         virtio_net_ctrl_ack status;
 124         struct virtio_net_ctrl_mq mq;
 125         u8 promisc;
 126         u8 allmulti;
 127         __virtio16 vid;
 128         u64 offloads;
 129 };
 130
 131 struct virtnet_info {
 132         struct virtio_device *vdev;
 133         struct virtqueue *cvq;
 134         struct net_device *dev;
 135         struct send_queue *sq;
 136         struct receive_queue *rq;
 137         unsigned int status;
 138
 139         /* Max # of queue pairs supported by the device */
 140         u16 max_queue_pairs;
 141
 142         /* # of queue pairs currently used by the driver */
 143         u16 curr_queue_pairs;
 144
 145         /* # of XDP queue pairs currently used by the driver */
 146         u16 xdp_queue_pairs;
 147
 148         /* I like... big packets and I cannot lie! */
 149         bool big_packets;
 150
 151         /* Host will merge rx buffers for big packets (shake it! shake it!) */
 152         bool mergeable_rx_bufs;
 153
 154         /* Has control virtqueue */
 155         bool has_cvq;
 156
 157         /* Host can handle any s/g split between our header and packet data */
 158         bool any_header_sg;
 159
 160         /* Packet virtio header size */
 161         u8 hdr_len;
 162
 163         /* Active statistics */
 164         struct virtnet_stats __percpu *stats;
 165
 166         /* Work struct for refilling if we run low on memory. */
 167         struct delayed_work refill;
 168
 169         /* Work struct for config space updates */
 170         struct work_struct config_work;
 171
 172         /* Does the affinity hint is set for virtqueues? */
 173         bool affinity_hint_set;
 174
 175         /* CPU hotplug instances for online & dead */
 176         struct hlist_node node;
 177         struct hlist_node node_dead;
 178
 179         struct control_buf *ctrl;
 180
 181         /* Ethtool settings */
 182         u8 duplex;
 183         u32 speed;
 184
 185         unsigned long guest_offloads;
 186 };
 187
 188 struct padded_vnet_hdr {
 189         struct virtio_net_hdr_mrg_rxbuf hdr;
 190         /*
 191          * hdr is in a separate sg buffer, and data sg buffer shares same page
 192          * with this header sg. This padding makes next sg 16 byte aligned
 193          * after the header.
 194          */
 195         char padding[4];
 196 };
 197
 198 /* Converting between virtqueue no. and kernel tx/rx queue no.
 199  * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 200  */
 201 static int vq2txq(struct virtqueue *vq)
 202 {
 203         return (vq->index - 1) / 2;
 204 }
 205
 206 static int txq2vq(int txq)
 207 {
 208         return txq * 2 + 1;
 209 }
 210
 211 static int vq2rxq(struct virtqueue *vq)
 212 {
 213         return vq->index / 2;
 214 }
 215
 216 static int rxq2vq(int rxq)
 217 {
 218         return rxq * 2;
 219 }
 220
 221 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
 222 {
 223         return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
 224 }
 225
 226 /*
 227  * private is used to chain pages for big packets, put the whole
 228  * most recent used list in the beginning for reuse
 229  */
 230 static void give_pages(struct receive_queue *rq, struct page *page)
 231 {
 232         struct page *end;
 233
 234         /* Find end of list, sew whole thing into vi->rq.pages. */
 235         for (end = page; end->private; end = (struct page *)end->private);
 236         end->private = (unsigned long)rq->pages;
 237         rq->pages = page;
 238 }
 239
 240 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 241 {
 242         struct page *p = rq->pages;
 243
 244         if (p) {
 245                 rq->pages = (struct page *)p->private;
 246                 /* clear private here, it is used to chain pages */
 247                 p->private = 0;
 248         } else
 249                 p = alloc_page(gfp_mask);
 250         return p;
 251 }
 252
 253 static void virtqueue_napi_schedule(struct napi_struct *napi,
 254                                     struct virtqueue *vq)
 255 {
 256         if (napi_schedule_prep(napi)) {
 257                 virtqueue_disable_cb(vq);
 258                 __napi_schedule(napi);
 259         }
 260 }
 261
 262 static void virtqueue_napi_complete(struct napi_struct *napi,
 263                                     struct virtqueue *vq, int processed)
 264 {
 265         int opaque;
 266
 267         opaque = virtqueue_enable_cb_prepare(vq);
 268         if (napi_complete_done(napi, processed)) {
 269                 if (unlikely(virtqueue_poll(vq, opaque)))
 270                         virtqueue_napi_schedule(napi, vq);
 271         } else {
 272                 virtqueue_disable_cb(vq);
 273         }
 274 }
 275
 276 static void skb_xmit_done(struct virtqueue *vq)
 277 {
 278         struct virtnet_info *vi = vq->vdev->priv;
 279         struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
 280
 281         /* Suppress further interrupts. */
 282         virtqueue_disable_cb(vq);
 283
 284         if (napi->weight)
 285                 virtqueue_napi_schedule(napi, vq);
 286         else
 287                 /* We were probably waiting for more output buffers. */
 288                 netif_wake_subqueue(vi->dev, vq2txq(vq));
 289 }
 290
 291 #define MRG_CTX_HEADER_SHIFT 22
 292 static void *mergeable_len_to_ctx(unsigned int truesize,
 293                                   unsigned int headroom)
 294 {
 295         return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
 296 }
 297
 298 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
 299 {
 300         return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
 301 }
 302
 303 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
 304 {
 305         return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
 306 }
 307
 308 /* Called from bottom half context */
 309 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 310                                    struct receive_queue *rq,
 311                                    struct page *page, unsigned int offset,
 312                                    unsigned int len, unsigned int truesize,
 313                                    bool hdr_valid)
 314 {
 315         struct sk_buff *skb;
 316         struct virtio_net_hdr_mrg_rxbuf *hdr;
 317         unsigned int copy, hdr_len, hdr_padded_len;
 318         char *p;
 319
 320         p = page_address(page) + offset;
 321
 322         /* copy small packet so we can reuse these pages for small data */
 323         skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
 324         if (unlikely(!skb))
 325                 return NULL;
 326
 327         hdr = skb_vnet_hdr(skb);
 328
 329         hdr_len = vi->hdr_len;
 330         if (vi->mergeable_rx_bufs)
 331                 hdr_padded_len = sizeof(*hdr);
 332         else
 333                 hdr_padded_len = sizeof(struct padded_vnet_hdr);
 334
 335         if (hdr_valid)
 336                 memcpy(hdr, p, hdr_len);
 337
 338         len -= hdr_len;
 339         offset += hdr_padded_len;
 340         p += hdr_padded_len;
 341
 342         /* Copy all frame if it fits skb->head, otherwise
 343          * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
 344          */
 345         if (len <= skb_tailroom(skb))
 346                 copy = len;
 347         else
 348                 copy = ETH_HLEN;
 349         skb_put_data(skb, p, copy);
 350
 351         len -= copy;
 352         offset += copy;
 353
 354         if (vi->mergeable_rx_bufs) {
 355                 if (len)
 356                         skb_add_rx_frag(skb, 0, page, offset, len, truesize);
 357                 else
 358                         put_page(page);
 359                 return skb;
 360         }
 361
 362         /*
 363          * Verify that we can indeed put this data into a skb.
 364          * This is here to handle cases when the device erroneously
 365          * tries to receive more than is possible. This is usually
 366          * the case of a broken device.
 367          */
 368         if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
 369                 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
 370                 dev_kfree_skb(skb);
 371                 return NULL;
 372         }
 373         BUG_ON(offset >= PAGE_SIZE);
 374         while (len) {
 375                 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
 376                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
 377                                 frag_size, truesize);
 378                 len -= frag_size;
 379                 page = (struct page *)page->private;
 380                 offset = 0;
 381         }
 382
 383         if (page)
 384                 give_pages(rq, page);
 385
 386         return skb;
 387 }
 388
 389 static bool virtnet_xdp_xmit(struct virtnet_info *vi,
 390                              struct receive_queue *rq,
 391                              struct xdp_buff *xdp)
 392 {
 393         struct virtio_net_hdr_mrg_rxbuf *hdr;
 394         unsigned int len;
 395         struct send_queue *sq;
 396         unsigned int qp;
 397         void *xdp_sent;
 398         int err;
 399
 400         qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
 401         sq = &vi->sq[qp];
 402
 403         /* Free up any pending old buffers before queueing new ones. */
 404         while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
 405                 struct page *sent_page = virt_to_head_page(xdp_sent);
 406
 407                 put_page(sent_page);
 408         }
 409
 410         xdp->data -= vi->hdr_len;
 411         /* Zero header and leave csum up to XDP layers */
 412         hdr = xdp->data;
 413         memset(hdr, 0, vi->hdr_len);
 414
 415         sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
 416
 417         err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
 418         if (unlikely(err)) {
 419                 struct page *page = virt_to_head_page(xdp->data);
 420
 421                 put_page(page);
 422                 return false;
 423         }
 424
 425         virtqueue_kick(sq->vq);
 426         return true;
 427 }
 428
 429 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
 430 {
 431         return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
 432 }
 433
 434 /* We copy the packet for XDP in the following cases:
 435  *
 436  * 1) Packet is scattered across multiple rx buffers.
 437  * 2) Headroom space is insufficient.
 438  *
 439  * This is inefficient but it's a temporary condition that
 440  * we hit right after XDP is enabled and until queue is refilled
 441  * with large buffers with sufficient headroom - so it should affect
 442  * at most queue size packets.
 443  * Afterwards, the conditions to enable
 444  * XDP should preclude the underlying device from sending packets
 445  * across multiple buffers (num_buf > 1), and we make sure buffers
 446  * have enough headroom.
 447  */
 448 static struct page *xdp_linearize_page(struct receive_queue *rq,
 449                                        u16 *num_buf,
 450                                        struct page *p,
 451                                        int offset,
 452                                        int page_off,
 453                                        unsigned int *len)
 454 {
 455         int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 456         struct page *page;
 457
 458         if (page_off + *len + tailroom > PAGE_SIZE)
 459                 return NULL;
 460
 461         page = alloc_page(GFP_ATOMIC);
 462         if (!page)
 463                 return NULL;
 464
 465         memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
 466         page_off += *len;
 467
 468         while (--*num_buf) {
 469                 unsigned int buflen;
 470                 void *buf;
 471                 int off;
 472
 473                 buf = virtqueue_get_buf(rq->vq, &buflen);
 474                 if (unlikely(!buf))
 475                         goto err_buf;
 476
 477                 p = virt_to_head_page(buf);
 478                 off = buf - page_address(p);
 479
 480                 /* guard against a misconfigured or uncooperative backend that
 481                  * is sending packet larger than the MTU.
 482                  */
 483                 if ((page_off + buflen) > PAGE_SIZE) {
 484                         put_page(p);
 485                         goto err_buf;
 486                 }
 487
 488                 memcpy(page_address(page) + page_off,
 489                        page_address(p) + off, buflen);
 490                 page_off += buflen;
 491                 put_page(p);
 492         }
 493
 494         /* Headroom does not contribute to packet length */
 495         *len = page_off - VIRTIO_XDP_HEADROOM;
 496         return page;
 497 err_buf:
 498         __free_pages(page, 0);
 499         return NULL;
 500 }
 501
 502 static struct sk_buff *receive_small(struct net_device *dev,
 503                                      struct virtnet_info *vi,
 504                                      struct receive_queue *rq,
 505                                      void *buf, void *ctx,
 506                                      unsigned int len)
 507 {
 508         struct sk_buff *skb;
 509         struct bpf_prog *xdp_prog;
 510         unsigned int xdp_headroom = (unsigned long)ctx;
 511         unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
 512         unsigned int headroom = vi->hdr_len + header_offset;
 513         unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 514                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 515         struct page *page = virt_to_head_page(buf);
 516         unsigned int delta = 0;
 517         struct page *xdp_page;
 518         len -= vi->hdr_len;
 519
 520         rcu_read_lock();
 521         xdp_prog = rcu_dereference(rq->xdp_prog);
 522         if (xdp_prog) {
 523                 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
 524                 struct xdp_buff xdp;
 525                 void *orig_data;
 526                 u32 act;
 527
 528                 if (unlikely(hdr->hdr.gso_type))
 529                         goto err_xdp;
 530
 531                 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
 532                         int offset = buf - page_address(page) + header_offset;
 533                         unsigned int tlen = len + vi->hdr_len;
 534                         u16 num_buf = 1;
 535
 536                         xdp_headroom = virtnet_get_headroom(vi);
 537                         header_offset = VIRTNET_RX_PAD + xdp_headroom;
 538                         headroom = vi->hdr_len + header_offset;
 539                         buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 540                                  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 541                         xdp_page = xdp_linearize_page(rq, &num_buf, page,
 542                                                       offset, header_offset,
 543                                                       &tlen);
 544                         if (!xdp_page)
 545                                 goto err_xdp;
 546
 547                         buf = page_address(xdp_page);
 548                         put_page(page);
 549                         page = xdp_page;
 550                 }
 551
 552                 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
 553                 xdp.data = xdp.data_hard_start + xdp_headroom;
 554                 xdp.data_end = xdp.data + len;
 555                 orig_data = xdp.data;
 556                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 557
 558                 switch (act) {
 559                 case XDP_PASS:
 560                         /* Recalculate length in case bpf program changed it */
 561                         delta = orig_data - xdp.data;
 562                         break;
 563                 case XDP_TX:
 564                         if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
 565                                 trace_xdp_exception(vi->dev, xdp_prog, act);
 566                         rcu_read_unlock();
 567                         goto xdp_xmit;
 568                 default:
 569                         bpf_warn_invalid_xdp_action(act);
 570                 case XDP_ABORTED:
 571                         trace_xdp_exception(vi->dev, xdp_prog, act);
 572                 case XDP_DROP:
 573                         goto err_xdp;
 574                 }
 575         }
 576         rcu_read_unlock();
 577
 578         skb = build_skb(buf, buflen);
 579         if (!skb) {
 580                 put_page(page);
 581                 goto err;
 582         }
 583         skb_reserve(skb, headroom - delta);
 584         skb_put(skb, len + delta);
 585         if (!delta) {
 586                 buf += header_offset;
 587                 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
 588         } /* keep zeroed vnet hdr since packet was changed by bpf */
 589
 590 err:
 591         return skb;
 592
 593 err_xdp:
 594         rcu_read_unlock();
 595         dev->stats.rx_dropped++;
 596         put_page(page);
 597 xdp_xmit:
 598         return NULL;
 599 }
 600
 601 static struct sk_buff *receive_big(struct net_device *dev,
 602                                    struct virtnet_info *vi,
 603                                    struct receive_queue *rq,
 604                                    void *buf,
 605                                    unsigned int len)
 606 {
 607         struct page *page = buf;
 608         struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len,
 609                                           PAGE_SIZE, true);
 610
 611         if (unlikely(!skb))
 612                 goto err;
 613
 614         return skb;
 615
 616 err:
 617         dev->stats.rx_dropped++;
 618         give_pages(rq, page);
 619         return NULL;
 620 }
 621
 622 static struct sk_buff *receive_mergeable(struct net_device *dev,
 623                                          struct virtnet_info *vi,
 624                                          struct receive_queue *rq,
 625                                          void *buf,
 626                                          void *ctx,
 627                                          unsigned int len)
 628 {
 629         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
 630         u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
 631         struct page *page = virt_to_head_page(buf);
 632         int offset = buf - page_address(page);
 633         struct sk_buff *head_skb, *curr_skb;
 634         struct bpf_prog *xdp_prog;
 635         unsigned int truesize;
 636         unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 637
 638         head_skb = NULL;
 639
 640         rcu_read_lock();
 641         xdp_prog = rcu_dereference(rq->xdp_prog);
 642         if (xdp_prog) {
 643                 struct page *xdp_page;
 644                 struct xdp_buff xdp;
 645                 void *data;
 646                 u32 act;
 647
 648                 /* Transient failure which in theory could occur if
 649                  * in-flight packets from before XDP was enabled reach
 650                  * the receive path after XDP is loaded.
 651                  */
 652                 if (unlikely(hdr->hdr.gso_type))
 653                         goto err_xdp;
 654
 655                 /* This happens when rx buffer size is underestimated */
 656                 if (unlikely(num_buf > 1 ||
 657                              headroom < virtnet_get_headroom(vi))) {
 658                         /* linearize data for XDP */
 659                         xdp_page = xdp_linearize_page(rq, &num_buf,
 660                                                       page, offset,
 661                                                       VIRTIO_XDP_HEADROOM,
 662                                                       &len);
 663                         if (!xdp_page)
 664                                 goto err_xdp;
 665                         offset = VIRTIO_XDP_HEADROOM;
 666                 } else {
 667                         xdp_page = page;
 668                 }
 669
 670                 /* Allow consuming headroom but reserve enough space to push
 671                  * the descriptor on if we get an XDP_TX return code.
 672                  */
 673                 data = page_address(xdp_page) + offset;
 674                 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
 675                 xdp.data = data + vi->hdr_len;
 676                 xdp.data_end = xdp.data + (len - vi->hdr_len);
 677                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 678
 679                 switch (act) {
 680                 case XDP_PASS:
 681                         /* recalculate offset to account for any header
 682                          * adjustments. Note other cases do not build an
 683                          * skb and avoid using offset
 684                          */
 685                         offset = xdp.data -
 686                                         page_address(xdp_page) - vi->hdr_len;
 687
 688                         /* We can only create skb based on xdp_page. */
 689                         if (unlikely(xdp_page != page)) {
 690                                 rcu_read_unlock();
 691                                 put_page(page);
 692                                 head_skb = page_to_skb(vi, rq, xdp_page,
 693                                                        offset, len,
 694                                                        PAGE_SIZE, false);
 695                                 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 696                                 return head_skb;
 697                         }
 698                         break;
 699                 case XDP_TX:
 700                         if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
 701                                 trace_xdp_exception(vi->dev, xdp_prog, act);
 702                         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 703                         if (unlikely(xdp_page != page))
 704                                 put_page(page);
 705                         rcu_read_unlock();
 706                         goto xdp_xmit;
 707                 default:
 708                         bpf_warn_invalid_xdp_action(act);
 709                 case XDP_ABORTED:
 710                         trace_xdp_exception(vi->dev, xdp_prog, act);
 711                 case XDP_DROP:
 712                         if (unlikely(xdp_page != page))
 713                                 __free_pages(xdp_page, 0);
 714                         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 715                         goto err_xdp;
 716                 }
 717         }
 718         rcu_read_unlock();
 719
 720         truesize = mergeable_ctx_to_truesize(ctx);
 721         if (unlikely(len > truesize)) {
 722                 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 723                          dev->name, len, (unsigned long)ctx);
 724                 dev->stats.rx_length_errors++;
 725                 goto err_skb;
 726         }
 727
 728         head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog);
 729         curr_skb = head_skb;
 730
 731         if (unlikely(!curr_skb))
 732                 goto err_skb;
 733         while (--num_buf) {
 734                 int num_skb_frags;
 735
 736                 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
 737                 if (unlikely(!buf)) {
 738                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
 739                                  dev->name, num_buf,
 740                                  virtio16_to_cpu(vi->vdev,
 741                                                  hdr->num_buffers));
 742                         dev->stats.rx_length_errors++;
 743                         goto err_buf;
 744                 }
 745
 746                 page = virt_to_head_page(buf);
 747
 748                 truesize = mergeable_ctx_to_truesize(ctx);
 749                 if (unlikely(len > truesize)) {
 750                         pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 751                                  dev->name, len, (unsigned long)ctx);
 752                         dev->stats.rx_length_errors++;
 753                         goto err_skb;
 754                 }
 755
 756                 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
 757                 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
 758                         struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
 759
 760                         if (unlikely(!nskb))
 761                                 goto err_skb;
 762                         if (curr_skb == head_skb)
 763                                 skb_shinfo(curr_skb)->frag_list = nskb;
 764                         else
 765                                 curr_skb->next = nskb;
 766                         curr_skb = nskb;
 767                         head_skb->truesize += nskb->truesize;
 768                         num_skb_frags = 0;
 769                 }
 770                 if (curr_skb != head_skb) {
 771                         head_skb->data_len += len;
 772                         head_skb->len += len;
 773                         head_skb->truesize += truesize;
 774                 }
 775                 offset = buf - page_address(page);
 776                 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
 777                         put_page(page);
 778                         skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
 779                                              len, truesize);
 780                 } else {
 781                         skb_add_rx_frag(curr_skb, num_skb_frags, page,
 782                                         offset, len, truesize);
 783                 }
 784         }
 785
 786         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
 787         return head_skb;
 788
 789 err_xdp:
 790         rcu_read_unlock();
 791 err_skb:
 792         put_page(page);
 793         while (num_buf-- > 1) {
 794                 buf = virtqueue_get_buf(rq->vq, &len);
 795                 if (unlikely(!buf)) {
 796                         pr_debug("%s: rx error: %d buffers missing\n",
 797                                  dev->name, num_buf);
 798                         dev->stats.rx_length_errors++;
 799                         break;
 800                 }
 801                 page = virt_to_head_page(buf);
 802                 put_page(page);
 803         }
 804 err_buf:
 805         dev->stats.rx_dropped++;
 806         dev_kfree_skb(head_skb);
 807 xdp_xmit:
 808         return NULL;
 809 }
 810
 811 static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
 812                        void *buf, unsigned int len, void **ctx)
 813 {
 814         struct net_device *dev = vi->dev;
 815         struct sk_buff *skb;
 816         struct virtio_net_hdr_mrg_rxbuf *hdr;
 817         int ret;
 818
 819         if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
 820                 pr_debug("%s: short packet %i\n", dev->name, len);
 821                 dev->stats.rx_length_errors++;
 822                 if (vi->mergeable_rx_bufs) {
 823                         put_page(virt_to_head_page(buf));
 824                 } else if (vi->big_packets) {
 825                         give_pages(rq, buf);
 826                 } else {
 827                         put_page(virt_to_head_page(buf));
 828                 }
 829                 return 0;
 830         }
 831
 832         if (vi->mergeable_rx_bufs)
 833                 skb = receive_mergeable(dev, vi, rq, buf, ctx, len);
 834         else if (vi->big_packets)
 835                 skb = receive_big(dev, vi, rq, buf, len);
 836         else
 837                 skb = receive_small(dev, vi, rq, buf, ctx, len);
 838
 839         if (unlikely(!skb))
 840                 return 0;
 841
 842         hdr = skb_vnet_hdr(skb);
 843
 844         ret = skb->len;
 845
 846         if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
 847                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 848
 849         if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
 850                                   virtio_is_little_endian(vi->vdev))) {
 851                 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
 852                                      dev->name, hdr->hdr.gso_type,
 853                                      hdr->hdr.gso_size);
 854                 goto frame_err;
 855         }
 856
 857         skb->protocol = eth_type_trans(skb, dev);
 858         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
 859                  ntohs(skb->protocol), skb->len, skb->pkt_type);
 860
 861         napi_gro_receive(&rq->napi, skb);
 862         return ret;
 863
 864 frame_err:
 865         dev->stats.rx_frame_errors++;
 866         dev_kfree_skb(skb);
 867         return 0;
 868 }
 869
 870 /* Unlike mergeable buffers, all buffers are allocated to the
 871  * same size, except for the headroom. For this reason we do
 872  * not need to use  mergeable_len_to_ctx here - it is enough
 873  * to store the headroom as the context ignoring the truesize.
 874  */
 875 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
 876                              gfp_t gfp)
 877 {
 878         struct page_frag *alloc_frag = &rq->alloc_frag;
 879         char *buf;
 880         unsigned int xdp_headroom = virtnet_get_headroom(vi);
 881         void *ctx = (void *)(unsigned long)xdp_headroom;
 882         int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
 883         int err;
 884
 885         len = SKB_DATA_ALIGN(len) +
 886               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 887         if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
 888                 return -ENOMEM;
 889
 890         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 891         get_page(alloc_frag->page);
 892         alloc_frag->offset += len;
 893         sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
 894                     vi->hdr_len + GOOD_PACKET_LEN);
 895         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 896         if (err < 0)
 897                 put_page(virt_to_head_page(buf));
 898         return err;
 899 }
 900
 901 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
 902                            gfp_t gfp)
 903 {
 904         struct page *first, *list = NULL;
 905         char *p;
 906         int i, err, offset;
 907
 908         sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
 909
 910         /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
 911         for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
 912                 first = get_a_page(rq, gfp);
 913                 if (!first) {
 914                         if (list)
 915                                 give_pages(rq, list);
 916                         return -ENOMEM;
 917                 }
 918                 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
 919
 920                 /* chain new page in list head to match sg */
 921                 first->private = (unsigned long)list;
 922                 list = first;
 923         }
 924
 925         first = get_a_page(rq, gfp);
 926         if (!first) {
 927                 give_pages(rq, list);
 928                 return -ENOMEM;
 929         }
 930         p = page_address(first);
 931
 932         /* rq->sg[0], rq->sg[1] share the same page */
 933         /* a separated rq->sg[0] for header - required in case !any_header_sg */
 934         sg_set_buf(&rq->sg[0], p, vi->hdr_len);
 935
 936         /* rq->sg[1] for data packet, from offset */
 937         offset = sizeof(struct padded_vnet_hdr);
 938         sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
 939
 940         /* chain first in list head */
 941         first->private = (unsigned long)list;
 942         err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
 943                                   first, gfp);
 944         if (err < 0)
 945                 give_pages(rq, first);
 946
 947         return err;
 948 }
 949
 950 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
 951                                           struct ewma_pkt_len *avg_pkt_len)
 952 {
 953         const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 954         unsigned int len;
 955
 956         len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
 957                                 rq->min_buf_len, PAGE_SIZE - hdr_len);
 958         return ALIGN(len, L1_CACHE_BYTES);
 959 }
 960
 961 static int add_recvbuf_mergeable(struct virtnet_info *vi,
 962                                  struct receive_queue *rq, gfp_t gfp)
 963 {
 964         struct page_frag *alloc_frag = &rq->alloc_frag;
 965         unsigned int headroom = virtnet_get_headroom(vi);
 966         char *buf;
 967         void *ctx;
 968         int err;
 969         unsigned int len, hole;
 970
 971         len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len);
 972         if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
 973                 return -ENOMEM;
 974
 975         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 976         buf += headroom; /* advance address leaving hole at front of pkt */
 977         get_page(alloc_frag->page);
 978         alloc_frag->offset += len + headroom;
 979         hole = alloc_frag->size - alloc_frag->offset;
 980         if (hole < len + headroom) {
 981                 /* To avoid internal fragmentation, if there is very likely not
 982                  * enough space for another buffer, add the remaining space to
 983                  * the current buffer.
 984                  */
 985                 len += hole;
 986                 alloc_frag->offset += hole;
 987         }
 988
 989         sg_init_one(rq->sg, buf, len);
 990         ctx = mergeable_len_to_ctx(len, headroom);
 991         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 992         if (err < 0)
 993                 put_page(virt_to_head_page(buf));
 994
 995         return err;
 996 }
 997
 998 /*
 999  * Returns false if we couldn't fill entirely (OOM).
1000  *
1001  * Normally run in the receive path, but can also be run from ndo_open
1002  * before we're receiving packets, or from refill_work which is
1003  * careful to disable receiving (using napi_disable).
1004  */
1005 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1006                           gfp_t gfp)
1007 {
1008         int err;
1009         bool oom;
1010
1011         gfp |= __GFP_COLD;
1012         do {
1013                 if (vi->mergeable_rx_bufs)
1014                         err = add_recvbuf_mergeable(vi, rq, gfp);
1015                 else if (vi->big_packets)
1016                         err = add_recvbuf_big(vi, rq, gfp);
1017                 else
1018                         err = add_recvbuf_small(vi, rq, gfp);
1019
1020                 oom = err == -ENOMEM;
1021                 if (err)
1022                         break;
1023         } while (rq->vq->num_free);
1024         virtqueue_kick(rq->vq);
1025         return !oom;
1026 }
1027
1028 static void skb_recv_done(struct virtqueue *rvq)
1029 {
1030         struct virtnet_info *vi = rvq->vdev->priv;
1031         struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
1032
1033         virtqueue_napi_schedule(&rq->napi, rvq);
1034 }
1035
1036 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
1037 {
1038         napi_enable(napi);
1039
1040         /* If all buffers were filled by other side before we napi_enabled, we
1041          * won't get another interrupt, so process any outstanding packets now.
1042          * Call local_bh_enable after to trigger softIRQ processing.
1043          */
1044         local_bh_disable();
1045         virtqueue_napi_schedule(napi, vq);
1046         local_bh_enable();
1047 }
1048
1049 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1050                                    struct virtqueue *vq,
1051                                    struct napi_struct *napi)
1052 {
1053         if (!napi->weight)
1054                 return;
1055
1056         /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1057          * enable the feature if this is likely affine with the transmit path.
1058          */
1059         if (!vi->affinity_hint_set) {
1060                 napi->weight = 0;
1061                 return;
1062         }
1063
1064         return virtnet_napi_enable(vq, napi);
1065 }
1066
1067 static void virtnet_napi_tx_disable(struct napi_struct *napi)
1068 {
1069         if (napi->weight)
1070                 napi_disable(napi);
1071 }
1072
1073 static void refill_work(struct work_struct *work)
1074 {
1075         struct virtnet_info *vi =
1076                 container_of(work, struct virtnet_info, refill.work);
1077         bool still_empty;
1078         int i;
1079
1080         for (i = 0; i < vi->curr_queue_pairs; i++) {
1081                 struct receive_queue *rq = &vi->rq[i];
1082
1083                 napi_disable(&rq->napi);
1084                 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1085                 virtnet_napi_enable(rq->vq, &rq->napi);
1086
1087                 /* In theory, this can happen: if we don't get any buffers in
1088                  * we will *never* try to fill again.
1089                  */
1090                 if (still_empty)
1091                         schedule_delayed_work(&vi->refill, HZ/2);
1092         }
1093 }
1094
1095 static int virtnet_receive(struct receive_queue *rq, int budget)
1096 {
1097         struct virtnet_info *vi = rq->vq->vdev->priv;
1098         unsigned int len, received = 0, bytes = 0;
1099         void *buf;
1100         struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
1101
1102         if (!vi->big_packets || vi->mergeable_rx_bufs) {
1103                 void *ctx;
1104
1105                 while (received < budget &&
1106                        (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
1107                         bytes += receive_buf(vi, rq, buf, len, ctx);
1108                         received++;
1109                 }
1110         } else {
1111                 while (received < budget &&
1112                        (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1113                         bytes += receive_buf(vi, rq, buf, len, NULL);
1114                         received++;
1115                 }
1116         }
1117
1118         if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
1119                 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1120                         schedule_delayed_work(&vi->refill, 0);
1121         }
1122
1123         u64_stats_update_begin(&stats->rx_syncp);
1124         stats->rx_bytes += bytes;
1125         stats->rx_packets += received;
1126         u64_stats_update_end(&stats->rx_syncp);
1127
1128         return received;
1129 }
1130
1131 static void free_old_xmit_skbs(struct send_queue *sq)
1132 {
1133         struct sk_buff *skb;
1134         unsigned int len;
1135         struct virtnet_info *vi = sq->vq->vdev->priv;
1136         struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
1137         unsigned int packets = 0;
1138         unsigned int bytes = 0;
1139
1140         while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1141                 pr_debug("Sent skb %p\n", skb);
1142
1143                 bytes += skb->len;
1144                 packets++;
1145
1146                 dev_consume_skb_any(skb);
1147         }
1148
1149         /* Avoid overhead when no packets have been processed
1150          * happens when called speculatively from start_xmit.
1151          */
1152         if (!packets)
1153                 return;
1154
1155         u64_stats_update_begin(&stats->tx_syncp);
1156         stats->tx_bytes += bytes;
1157         stats->tx_packets += packets;
1158         u64_stats_update_end(&stats->tx_syncp);
1159 }
1160
1161 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1162 {
1163         if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1164                 return false;
1165         else if (q < vi->curr_queue_pairs)
1166                 return true;
1167         else
1168                 return false;
1169 }
1170
1171 static void virtnet_poll_cleantx(struct receive_queue *rq)
1172 {
1173         struct virtnet_info *vi = rq->vq->vdev->priv;
1174         unsigned int index = vq2rxq(rq->vq);
1175         struct send_queue *sq = &vi->sq[index];
1176         struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1177
1178         if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
1179                 return;
1180
1181         if (__netif_tx_trylock(txq)) {
1182                 free_old_xmit_skbs(sq);
1183                 __netif_tx_unlock(txq);
1184         }
1185
1186         if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1187                 netif_tx_wake_queue(txq);
1188 }
1189
1190 static int virtnet_poll(struct napi_struct *napi, int budget)
1191 {
1192         struct receive_queue *rq =
1193                 container_of(napi, struct receive_queue, napi);
1194         unsigned int received;
1195
1196         virtnet_poll_cleantx(rq);
1197
1198         received = virtnet_receive(rq, budget);
1199
1200         /* Out of packets? */
1201         if (received < budget)
1202                 virtqueue_napi_complete(napi, rq->vq, received);
1203
1204         return received;
1205 }
1206
1207 static int virtnet_open(struct net_device *dev)
1208 {
1209         struct virtnet_info *vi = netdev_priv(dev);
1210         int i;
1211
1212         for (i = 0; i < vi->max_queue_pairs; i++) {
1213                 if (i < vi->curr_queue_pairs)
1214                         /* Make sure we have some buffers: if oom use wq. */
1215                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1216                                 schedule_delayed_work(&vi->refill, 0);
1217                 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1218                 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
1219         }
1220
1221         return 0;
1222 }
1223
1224 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1225 {
1226         struct send_queue *sq = container_of(napi, struct send_queue, napi);
1227         struct virtnet_info *vi = sq->vq->vdev->priv;
1228         unsigned int index = vq2txq(sq->vq);
1229         struct netdev_queue *txq;
1230
1231         if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
1232                 /* We don't need to enable cb for XDP */
1233                 napi_complete_done(napi, 0);
1234                 return 0;
1235         }
1236
1237         txq = netdev_get_tx_queue(vi->dev, index);
1238         __netif_tx_lock(txq, raw_smp_processor_id());
1239         free_old_xmit_skbs(sq);
1240         __netif_tx_unlock(txq);
1241
1242         virtqueue_napi_complete(napi, sq->vq, 0);
1243
1244         if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1245                 netif_tx_wake_queue(txq);
1246
1247         return 0;
1248 }
1249
1250 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1251 {
1252         struct virtio_net_hdr_mrg_rxbuf *hdr;
1253         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1254         struct virtnet_info *vi = sq->vq->vdev->priv;
1255         int num_sg;
1256         unsigned hdr_len = vi->hdr_len;
1257         bool can_push;
1258
1259         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1260
1261         can_push = vi->any_header_sg &&
1262                 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1263                 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1264         /* Even if we can, don't push here yet as this would skew
1265          * csum_start offset below. */
1266         if (can_push)
1267                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1268         else
1269                 hdr = skb_vnet_hdr(skb);
1270
1271         if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1272                                     virtio_is_little_endian(vi->vdev), false,
1273                                     0))
1274                 return -EPROTO;
1275
1276         if (vi->mergeable_rx_bufs)
1277                 hdr->num_buffers = 0;
1278
1279         sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1280         if (can_push) {
1281                 __skb_push(skb, hdr_len);
1282                 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1283                 if (unlikely(num_sg < 0))
1284                         return num_sg;
1285                 /* Pull header back to avoid skew in tx bytes calculations. */
1286                 __skb_pull(skb, hdr_len);
1287         } else {
1288                 sg_set_buf(sq->sg, hdr, hdr_len);
1289                 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1290                 if (unlikely(num_sg < 0))
1291                         return num_sg;
1292                 num_sg++;
1293         }
1294         return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1295 }
1296
1297 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1298 {
1299         struct virtnet_info *vi = netdev_priv(dev);
1300         int qnum = skb_get_queue_mapping(skb);
1301         struct send_queue *sq = &vi->sq[qnum];
1302         int err;
1303         struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1304         bool kick = !skb->xmit_more;
1305         bool use_napi = sq->napi.weight;
1306
1307         /* Free up any pending old buffers before queueing new ones. */
1308         free_old_xmit_skbs(sq);
1309
1310         if (use_napi && kick)
1311                 virtqueue_enable_cb_delayed(sq->vq);
1312
1313         /* timestamp packet in software */
1314         skb_tx_timestamp(skb);
1315
1316         /* Try to transmit */
1317         err = xmit_skb(sq, skb);
1318
1319         /* This should not happen! */
1320         if (unlikely(err)) {
1321                 dev->stats.tx_fifo_errors++;
1322                 if (net_ratelimit())
1323                         dev_warn(&dev->dev,
1324                                  "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
1325                 dev->stats.tx_dropped++;
1326                 dev_kfree_skb_any(skb);
1327                 return NETDEV_TX_OK;
1328         }
1329
1330         /* Don't wait up for transmitted skbs to be freed. */
1331         if (!use_napi) {
1332                 skb_orphan(skb);
1333                 nf_reset(skb);
1334         }
1335
1336         /* If running out of space, stop queue to avoid getting packets that we
1337          * are then unable to transmit.
1338          * An alternative would be to force queuing layer to requeue the skb by
1339          * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1340          * returned in a normal path of operation: it means that driver is not
1341          * maintaining the TX queue stop/start state properly, and causes
1342          * the stack to do a non-trivial amount of useless work.
1343          * Since most packets only take 1 or 2 ring slots, stopping the queue
1344          * early means 16 slots are typically wasted.
1345          */
1346         if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1347                 netif_stop_subqueue(dev, qnum);
1348                 if (!use_napi &&
1349                     unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1350                         /* More just got used, free them then recheck. */
1351                         free_old_xmit_skbs(sq);
1352                         if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1353                                 netif_start_subqueue(dev, qnum);
1354                                 virtqueue_disable_cb(sq->vq);
1355                         }
1356                 }
1357         }
1358
1359         if (kick || netif_xmit_stopped(txq))
1360                 virtqueue_kick(sq->vq);
1361
1362         return NETDEV_TX_OK;
1363 }
1364
1365 /*
1366  * Send command via the control virtqueue and check status.  Commands
1367  * supported by the hypervisor, as indicated by feature bits, should
1368  * never fail unless improperly formatted.
1369  */
1370 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1371                                  struct scatterlist *out)
1372 {
1373         struct scatterlist *sgs[4], hdr, stat;
1374         unsigned out_num = 0, tmp;
1375
1376         /* Caller should know better */
1377         BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1378
1379         vi->ctrl->status = ~0;
1380         vi->ctrl->hdr.class = class;
1381         vi->ctrl->hdr.cmd = cmd;
1382         /* Add header */
1383         sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
1384         sgs[out_num++] = &hdr;
1385
1386         if (out)
1387                 sgs[out_num++] = out;
1388
1389         /* Add return status. */
1390         sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
1391         sgs[out_num] = &stat;
1392
1393         BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1394         virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1395
1396         if (unlikely(!virtqueue_kick(vi->cvq)))
1397                 return vi->ctrl->status == VIRTIO_NET_OK;
1398
1399         /* Spin for a response, the kick causes an ioport write, trapping
1400          * into the hypervisor, so the request should be handled immediately.
1401          */
1402         while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1403                !virtqueue_is_broken(vi->cvq))
1404                 cpu_relax();
1405
1406         return vi->ctrl->status == VIRTIO_NET_OK;
1407 }
1408
1409 static int virtnet_set_mac_address(struct net_device *dev, void *p)
1410 {
1411         struct virtnet_info *vi = netdev_priv(dev);
1412         struct virtio_device *vdev = vi->vdev;
1413         int ret;
1414         struct sockaddr *addr;
1415         struct scatterlist sg;
1416
1417         addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1418         if (!addr)
1419                 return -ENOMEM;
1420
1421         ret = eth_prepare_mac_addr_change(dev, addr);
1422         if (ret)
1423                 goto out;
1424
1425         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1426                 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1427                 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1428                                           VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1429                         dev_warn(&vdev->dev,
1430                                  "Failed to set mac address by vq command.\n");
1431                         ret = -EINVAL;
1432                         goto out;
1433                 }
1434         } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1435                    !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1436                 unsigned int i;
1437
1438                 /* Naturally, this has an atomicity problem. */
1439                 for (i = 0; i < dev->addr_len; i++)
1440                         virtio_cwrite8(vdev,
1441                                        offsetof(struct virtio_net_config, mac) +
1442                                        i, addr->sa_data[i]);
1443         }
1444
1445         eth_commit_mac_addr_change(dev, p);
1446         ret = 0;
1447
1448 out:
1449         kfree(addr);
1450         return ret;
1451 }
1452
1453 static void virtnet_stats(struct net_device *dev,
1454                           struct rtnl_link_stats64 *tot)
1455 {
1456         struct virtnet_info *vi = netdev_priv(dev);
1457         int cpu;
1458         unsigned int start;
1459
1460         for_each_possible_cpu(cpu) {
1461                 struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
1462                 u64 tpackets, tbytes, rpackets, rbytes;
1463
1464                 do {
1465                         start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
1466                         tpackets = stats->tx_packets;
1467                         tbytes   = stats->tx_bytes;
1468                 } while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
1469
1470                 do {
1471                         start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
1472                         rpackets = stats->rx_packets;
1473                         rbytes   = stats->rx_bytes;
1474                 } while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
1475
1476                 tot->rx_packets += rpackets;
1477                 tot->tx_packets += tpackets;
1478                 tot->rx_bytes   += rbytes;
1479                 tot->tx_bytes   += tbytes;
1480         }
1481
1482         tot->tx_dropped = dev->stats.tx_dropped;
1483         tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1484         tot->rx_dropped = dev->stats.rx_dropped;
1485         tot->rx_length_errors = dev->stats.rx_length_errors;
1486         tot->rx_frame_errors = dev->stats.rx_frame_errors;
1487 }
1488
1489 #ifdef CONFIG_NET_POLL_CONTROLLER
1490 static void virtnet_netpoll(struct net_device *dev)
1491 {
1492         struct virtnet_info *vi = netdev_priv(dev);
1493         int i;
1494
1495         for (i = 0; i < vi->curr_queue_pairs; i++)
1496                 napi_schedule(&vi->rq[i].napi);
1497 }
1498 #endif
1499
1500 static void virtnet_ack_link_announce(struct virtnet_info *vi)
1501 {
1502         rtnl_lock();
1503         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1504                                   VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1505                 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1506         rtnl_unlock();
1507 }
1508
1509 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1510 {
1511         struct scatterlist sg;
1512         struct net_device *dev = vi->dev;
1513
1514         if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1515                 return 0;
1516
1517         vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1518         sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
1519
1520         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1521                                   VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1522                 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1523                          queue_pairs);
1524                 return -EINVAL;
1525         } else {
1526                 vi->curr_queue_pairs = queue_pairs;
1527                 /* virtnet_open() will refill when device is going to up. */
1528                 if (dev->flags & IFF_UP)
1529                         schedule_delayed_work(&vi->refill, 0);
1530         }
1531
1532         return 0;
1533 }
1534
1535 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1536 {
1537         int err;
1538
1539         rtnl_lock();
1540         err = _virtnet_set_queues(vi, queue_pairs);
1541         rtnl_unlock();
1542         return err;
1543 }
1544
1545 static int virtnet_close(struct net_device *dev)
1546 {
1547         struct virtnet_info *vi = netdev_priv(dev);
1548         int i;
1549
1550         /* Make sure refill_work doesn't re-enable napi! */
1551         cancel_delayed_work_sync(&vi->refill);
1552
1553         for (i = 0; i < vi->max_queue_pairs; i++) {
1554                 napi_disable(&vi->rq[i].napi);
1555                 virtnet_napi_tx_disable(&vi->sq[i].napi);
1556         }
1557
1558         return 0;
1559 }
1560
1561 static void virtnet_set_rx_mode(struct net_device *dev)
1562 {
1563         struct virtnet_info *vi = netdev_priv(dev);
1564         struct scatterlist sg[2];
1565         struct virtio_net_ctrl_mac *mac_data;
1566         struct netdev_hw_addr *ha;
1567         int uc_count;
1568         int mc_count;
1569         void *buf;
1570         int i;
1571
1572         /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1573         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1574                 return;
1575
1576         vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
1577         vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1578
1579         sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
1580
1581         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1582                                   VIRTIO_NET_CTRL_RX_PROMISC, sg))
1583                 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1584                          vi->ctrl->promisc ? "en" : "dis");
1585
1586         sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
1587
1588         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1589                                   VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1590                 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1591                          vi->ctrl->allmulti ? "en" : "dis");
1592
1593         uc_count = netdev_uc_count(dev);
1594         mc_count = netdev_mc_count(dev);
1595         /* MAC filter - use one buffer for both lists */
1596         buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1597                       (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1598         mac_data = buf;
1599         if (!buf)
1600                 return;
1601
1602         sg_init_table(sg, 2);
1603
1604         /* Store the unicast list and count in the front of the buffer */
1605         mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
1606         i = 0;
1607         netdev_for_each_uc_addr(ha, dev)
1608                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1609
1610         sg_set_buf(&sg[0], mac_data,
1611                    sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
1612
1613         /* multicast list and count fill the end */
1614         mac_data = (void *)&mac_data->macs[uc_count][0];
1615
1616         mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
1617         i = 0;
1618         netdev_for_each_mc_addr(ha, dev)
1619                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1620
1621         sg_set_buf(&sg[1], mac_data,
1622                    sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
1623
1624         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1625                                   VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
1626                 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
1627
1628         kfree(buf);
1629 }
1630
1631 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
1632                                    __be16 proto, u16 vid)
1633 {
1634         struct virtnet_info *vi = netdev_priv(dev);
1635         struct scatterlist sg;
1636
1637         vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
1638         sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
1639
1640         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1641                                   VIRTIO_NET_CTRL_VLAN_ADD, &sg))
1642                 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
1643         return 0;
1644 }
1645
1646 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
1647                                     __be16 proto, u16 vid)
1648 {
1649         struct virtnet_info *vi = netdev_priv(dev);
1650         struct scatterlist sg;
1651
1652         vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
1653         sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
1654
1655         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1656                                   VIRTIO_NET_CTRL_VLAN_DEL, &sg))
1657                 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
1658         return 0;
1659 }
1660
1661 static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
1662 {
1663         int i;
1664
1665         if (vi->affinity_hint_set) {
1666                 for (i = 0; i < vi->max_queue_pairs; i++) {
1667                         virtqueue_set_affinity(vi->rq[i].vq, -1);
1668                         virtqueue_set_affinity(vi->sq[i].vq, -1);
1669                 }
1670
1671                 vi->affinity_hint_set = false;
1672         }
1673 }
1674
1675 static void virtnet_set_affinity(struct virtnet_info *vi)
1676 {
1677         int i;
1678         int cpu;
1679
1680         /* In multiqueue mode, when the number of cpu is equal to the number of
1681          * queue pairs, we let the queue pairs to be private to one cpu by
1682          * setting the affinity hint to eliminate the contention.
1683          */
1684         if (vi->curr_queue_pairs == 1 ||
1685             vi->max_queue_pairs != num_online_cpus()) {
1686                 virtnet_clean_affinity(vi, -1);
1687                 return;
1688         }
1689
1690         i = 0;
1691         for_each_online_cpu(cpu) {
1692                 virtqueue_set_affinity(vi->rq[i].vq, cpu);
1693                 virtqueue_set_affinity(vi->sq[i].vq, cpu);
1694                 netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
1695                 i++;
1696         }
1697
1698         vi->affinity_hint_set = true;
1699 }
1700
1701 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
1702 {
1703         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1704                                                    node);
1705         virtnet_set_affinity(vi);
1706         return 0;
1707 }
1708
1709 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
1710 {
1711         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1712                                                    node_dead);
1713         virtnet_set_affinity(vi);
1714         return 0;
1715 }
1716
1717 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
1718 {
1719         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1720                                                    node);
1721
1722         virtnet_clean_affinity(vi, cpu);
1723         return 0;
1724 }
1725
1726 static enum cpuhp_state virtionet_online;
1727
1728 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
1729 {
1730         int ret;
1731
1732         ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
1733         if (ret)
1734                 return ret;
1735         ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1736                                                &vi->node_dead);
1737         if (!ret)
1738                 return ret;
1739         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1740         return ret;
1741 }
1742
1743 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
1744 {
1745         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1746         cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1747                                             &vi->node_dead);
1748 }
1749
1750 static void virtnet_get_ringparam(struct net_device *dev,
1751                                 struct ethtool_ringparam *ring)
1752 {
1753         struct virtnet_info *vi = netdev_priv(dev);
1754
1755         ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
1756         ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
1757         ring->rx_pending = ring->rx_max_pending;
1758         ring->tx_pending = ring->tx_max_pending;
1759 }
1760
1761
1762 static void virtnet_get_drvinfo(struct net_device *dev,
1763                                 struct ethtool_drvinfo *info)
1764 {
1765         struct virtnet_info *vi = netdev_priv(dev);
1766         struct virtio_device *vdev = vi->vdev;
1767
1768         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1769         strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
1770         strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
1771
1772 }
1773
1774 /* TODO: Eliminate OOO packets during switching */
1775 static int virtnet_set_channels(struct net_device *dev,
1776                                 struct ethtool_channels *channels)
1777 {
1778         struct virtnet_info *vi = netdev_priv(dev);
1779         u16 queue_pairs = channels->combined_count;
1780         int err;
1781
1782         /* We don't support separate rx/tx channels.
1783          * We don't allow setting 'other' channels.
1784          */
1785         if (channels->rx_count || channels->tx_count || channels->other_count)
1786                 return -EINVAL;
1787
1788         if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
1789                 return -EINVAL;
1790
1791         /* For now we don't support modifying channels while XDP is loaded
1792          * also when XDP is loaded all RX queues have XDP programs so we only
1793          * need to check a single RX queue.
1794          */
1795         if (vi->rq[0].xdp_prog)
1796                 return -EINVAL;
1797
1798         get_online_cpus();
1799         err = _virtnet_set_queues(vi, queue_pairs);
1800         if (err) {
1801                 put_online_cpus();
1802                 goto err;
1803         }
1804         virtnet_set_affinity(vi);
1805         put_online_cpus();
1806
1807         netif_set_real_num_tx_queues(dev, queue_pairs);
1808         netif_set_real_num_rx_queues(dev, queue_pairs);
1809  err:
1810         return err;
1811 }
1812
1813 static void virtnet_get_channels(struct net_device *dev,
1814                                  struct ethtool_channels *channels)
1815 {
1816         struct virtnet_info *vi = netdev_priv(dev);
1817
1818         channels->combined_count = vi->curr_queue_pairs;
1819         channels->max_combined = vi->max_queue_pairs;
1820         channels->max_other = 0;
1821         channels->rx_count = 0;
1822         channels->tx_count = 0;
1823         channels->other_count = 0;
1824 }
1825
1826 /* Check if the user is trying to change anything besides speed/duplex */
1827 static bool
1828 virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd)
1829 {
1830         struct ethtool_link_ksettings diff1 = *cmd;
1831         struct ethtool_link_ksettings diff2 = {};
1832
1833         /* cmd is always set so we need to clear it, validate the port type
1834          * and also without autonegotiation we can ignore advertising
1835          */
1836         diff1.base.speed = 0;
1837         diff2.base.port = PORT_OTHER;
1838         ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
1839         diff1.base.duplex = 0;
1840         diff1.base.cmd = 0;
1841         diff1.base.link_mode_masks_nwords = 0;
1842
1843         return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) &&
1844                 bitmap_empty(diff1.link_modes.supported,
1845                              __ETHTOOL_LINK_MODE_MASK_NBITS) &&
1846                 bitmap_empty(diff1.link_modes.advertising,
1847                              __ETHTOOL_LINK_MODE_MASK_NBITS) &&
1848                 bitmap_empty(diff1.link_modes.lp_advertising,
1849                              __ETHTOOL_LINK_MODE_MASK_NBITS);
1850 }
1851
1852 static int virtnet_set_link_ksettings(struct net_device *dev,
1853                                       const struct ethtool_link_ksettings *cmd)
1854 {
1855         struct virtnet_info *vi = netdev_priv(dev);
1856         u32 speed;
1857
1858         speed = cmd->base.speed;
1859         /* don't allow custom speed and duplex */
1860         if (!ethtool_validate_speed(speed) ||
1861             !ethtool_validate_duplex(cmd->base.duplex) ||
1862             !virtnet_validate_ethtool_cmd(cmd))
1863                 return -EINVAL;
1864         vi->speed = speed;
1865         vi->duplex = cmd->base.duplex;
1866
1867         return 0;
1868 }
1869
1870 static int virtnet_get_link_ksettings(struct net_device *dev,
1871                                       struct ethtool_link_ksettings *cmd)
1872 {
1873         struct virtnet_info *vi = netdev_priv(dev);
1874
1875         cmd->base.speed = vi->speed;
1876         cmd->base.duplex = vi->duplex;
1877         cmd->base.port = PORT_OTHER;
1878
1879         return 0;
1880 }
1881
1882 static void virtnet_init_settings(struct net_device *dev)
1883 {
1884         struct virtnet_info *vi = netdev_priv(dev);
1885
1886         vi->speed = SPEED_UNKNOWN;
1887         vi->duplex = DUPLEX_UNKNOWN;
1888 }
1889
1890 static const struct ethtool_ops virtnet_ethtool_ops = {
1891         .get_drvinfo = virtnet_get_drvinfo,
1892         .get_link = ethtool_op_get_link,
1893         .get_ringparam = virtnet_get_ringparam,
1894         .set_channels = virtnet_set_channels,
1895         .get_channels = virtnet_get_channels,
1896         .get_ts_info = ethtool_op_get_ts_info,
1897         .get_link_ksettings = virtnet_get_link_ksettings,
1898         .set_link_ksettings = virtnet_set_link_ksettings,
1899 };
1900
1901 static void virtnet_freeze_down(struct virtio_device *vdev)
1902 {
1903         struct virtnet_info *vi = vdev->priv;
1904         int i;
1905
1906         /* Make sure no work handler is accessing the device */
1907         flush_work(&vi->config_work);
1908
1909         netif_tx_lock_bh(vi->dev);
1910         netif_device_detach(vi->dev);
1911         netif_tx_unlock_bh(vi->dev);
1912         cancel_delayed_work_sync(&vi->refill);
1913
1914         if (netif_running(vi->dev)) {
1915                 for (i = 0; i < vi->max_queue_pairs; i++) {
1916                         napi_disable(&vi->rq[i].napi);
1917                         virtnet_napi_tx_disable(&vi->sq[i].napi);
1918                 }
1919         }
1920 }
1921
1922 static int init_vqs(struct virtnet_info *vi);
1923
1924 static int virtnet_restore_up(struct virtio_device *vdev)
1925 {
1926         struct virtnet_info *vi = vdev->priv;
1927         int err, i;
1928
1929         err = init_vqs(vi);
1930         if (err)
1931                 return err;
1932
1933         virtio_device_ready(vdev);
1934
1935         if (netif_running(vi->dev)) {
1936                 for (i = 0; i < vi->curr_queue_pairs; i++)
1937                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1938                                 schedule_delayed_work(&vi->refill, 0);
1939
1940                 for (i = 0; i < vi->max_queue_pairs; i++) {
1941                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1942                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
1943                                                &vi->sq[i].napi);
1944                 }
1945         }
1946
1947         netif_tx_lock_bh(vi->dev);
1948         netif_device_attach(vi->dev);
1949         netif_tx_unlock_bh(vi->dev);
1950         return err;
1951 }
1952
1953 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
1954 {
1955         struct scatterlist sg;
1956         vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
1957
1958         sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
1959
1960         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
1961                                   VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
1962                 dev_warn(&vi->dev->dev, "Fail to set guest offload. \n");
1963                 return -EINVAL;
1964         }
1965
1966         return 0;
1967 }
1968
1969 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
1970 {
1971         u64 offloads = 0;
1972
1973         if (!vi->guest_offloads)
1974                 return 0;
1975
1976         return virtnet_set_guest_offloads(vi, offloads);
1977 }
1978
1979 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
1980 {
1981         u64 offloads = vi->guest_offloads;
1982
1983         if (!vi->guest_offloads)
1984                 return 0;
1985
1986         return virtnet_set_guest_offloads(vi, offloads);
1987 }
1988
1989 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1990                            struct netlink_ext_ack *extack)
1991 {
1992         unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
1993         struct virtnet_info *vi = netdev_priv(dev);
1994         struct bpf_prog *old_prog;
1995         u16 xdp_qp = 0, curr_qp;
1996         int i, err;
1997
1998         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
1999             && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2000                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2001                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2002                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
2003                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {
2004                 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO/CSUM, disable LRO/CSUM first");
2005                 return -EOPNOTSUPP;
2006         }
2007
2008         if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
2009                 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
2010                 return -EINVAL;
2011         }
2012
2013         if (dev->mtu > max_sz) {
2014                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
2015                 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2016                 return -EINVAL;
2017         }
2018
2019         curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2020         if (prog)
2021                 xdp_qp = nr_cpu_ids;
2022
2023         /* XDP requires extra queues for XDP_TX */
2024         if (curr_qp + xdp_qp > vi->max_queue_pairs) {
2025                 NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available");
2026                 netdev_warn(dev, "request %i queues but max is %i\n",
2027                             curr_qp + xdp_qp, vi->max_queue_pairs);
2028                 return -ENOMEM;
2029         }
2030
2031         if (prog) {
2032                 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
2033                 if (IS_ERR(prog))
2034                         return PTR_ERR(prog);
2035         }
2036
2037         /* Make sure NAPI is not using any XDP TX queues for RX. */
2038         if (netif_running(dev)) {
2039                 for (i = 0; i < vi->max_queue_pairs; i++) {
2040                         napi_disable(&vi->rq[i].napi);
2041                         virtnet_napi_tx_disable(&vi->sq[i].napi);
2042                 }
2043         }
2044
2045         err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2046         if (err)
2047                 goto err;
2048         netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
2049         vi->xdp_queue_pairs = xdp_qp;
2050
2051         for (i = 0; i < vi->max_queue_pairs; i++) {
2052                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2053                 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2054                 if (i == 0) {
2055                         if (!old_prog)
2056                                 virtnet_clear_guest_offloads(vi);
2057                         if (!prog)
2058                                 virtnet_restore_guest_offloads(vi);
2059                 }
2060                 if (old_prog)
2061                         bpf_prog_put(old_prog);
2062                 if (netif_running(dev)) {
2063                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2064                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2065                                                &vi->sq[i].napi);
2066                 }
2067         }
2068
2069         return 0;
2070
2071 err:
2072         if (netif_running(dev)) {
2073                 for (i = 0; i < vi->max_queue_pairs; i++) {
2074                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2075                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2076                                                &vi->sq[i].napi);
2077                 }
2078         }
2079         if (prog)
2080                 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2081         return err;
2082 }
2083
2084 static u32 virtnet_xdp_query(struct net_device *dev)
2085 {
2086         struct virtnet_info *vi = netdev_priv(dev);
2087         const struct bpf_prog *xdp_prog;
2088         int i;
2089
2090         for (i = 0; i < vi->max_queue_pairs; i++) {
2091                 xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2092                 if (xdp_prog)
2093                         return xdp_prog->aux->id;
2094         }
2095         return 0;
2096 }
2097
2098 static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp)
2099 {
2100         switch (xdp->command) {
2101         case XDP_SETUP_PROG:
2102                 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
2103         case XDP_QUERY_PROG:
2104                 xdp->prog_id = virtnet_xdp_query(dev);
2105                 xdp->prog_attached = !!xdp->prog_id;
2106                 return 0;
2107         default:
2108                 return -EINVAL;
2109         }
2110 }
2111
2112 static const struct net_device_ops virtnet_netdev = {
2113         .ndo_open            = virtnet_open,
2114         .ndo_stop            = virtnet_close,
2115         .ndo_start_xmit      = start_xmit,
2116         .ndo_validate_addr   = eth_validate_addr,
2117         .ndo_set_mac_address = virtnet_set_mac_address,
2118         .ndo_set_rx_mode     = virtnet_set_rx_mode,
2119         .ndo_get_stats64     = virtnet_stats,
2120         .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
2121         .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
2122 #ifdef CONFIG_NET_POLL_CONTROLLER
2123         .ndo_poll_controller = virtnet_netpoll,
2124 #endif
2125         .ndo_xdp                = virtnet_xdp,
2126         .ndo_features_check     = passthru_features_check,
2127 };
2128
2129 static void virtnet_config_changed_work(struct work_struct *work)
2130 {
2131         struct virtnet_info *vi =
2132                 container_of(work, struct virtnet_info, config_work);
2133         u16 v;
2134
2135         if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2136                                  struct virtio_net_config, status, &v) < 0)
2137                 return;
2138
2139         if (v & VIRTIO_NET_S_ANNOUNCE) {
2140                 netdev_notify_peers(vi->dev);
2141                 virtnet_ack_link_announce(vi);
2142         }
2143
2144         /* Ignore unknown (future) status bits */
2145         v &= VIRTIO_NET_S_LINK_UP;
2146
2147         if (vi->status == v)
2148                 return;
2149
2150         vi->status = v;
2151
2152         if (vi->status & VIRTIO_NET_S_LINK_UP) {
2153                 netif_carrier_on(vi->dev);
2154                 netif_tx_wake_all_queues(vi->dev);
2155         } else {
2156                 netif_carrier_off(vi->dev);
2157                 netif_tx_stop_all_queues(vi->dev);
2158         }
2159 }
2160
2161 static void virtnet_config_changed(struct virtio_device *vdev)
2162 {
2163         struct virtnet_info *vi = vdev->priv;
2164
2165         schedule_work(&vi->config_work);
2166 }
2167
2168 static void virtnet_free_queues(struct virtnet_info *vi)
2169 {
2170         int i;
2171
2172         for (i = 0; i < vi->max_queue_pairs; i++) {
2173                 napi_hash_del(&vi->rq[i].napi);
2174                 netif_napi_del(&vi->rq[i].napi);
2175                 netif_napi_del(&vi->sq[i].napi);
2176         }
2177
2178         /* We called napi_hash_del() before netif_napi_del(),
2179          * we need to respect an RCU grace period before freeing vi->rq
2180          */
2181         synchronize_net();
2182
2183         kfree(vi->rq);
2184         kfree(vi->sq);
2185         kfree(vi->ctrl);
2186 }
2187
2188 static void _free_receive_bufs(struct virtnet_info *vi)
2189 {
2190         struct bpf_prog *old_prog;
2191         int i;
2192
2193         for (i = 0; i < vi->max_queue_pairs; i++) {
2194                 while (vi->rq[i].pages)
2195                         __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
2196
2197                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2198                 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2199                 if (old_prog)
2200                         bpf_prog_put(old_prog);
2201         }
2202 }
2203
2204 static void free_receive_bufs(struct virtnet_info *vi)
2205 {
2206         rtnl_lock();
2207         _free_receive_bufs(vi);
2208         rtnl_unlock();
2209 }
2210
2211 static void free_receive_page_frags(struct virtnet_info *vi)
2212 {
2213         int i;
2214         for (i = 0; i < vi->max_queue_pairs; i++)
2215                 if (vi->rq[i].alloc_frag.page)
2216                         put_page(vi->rq[i].alloc_frag.page);
2217 }
2218
2219 static void free_unused_bufs(struct virtnet_info *vi)
2220 {
2221         void *buf;
2222         int i;
2223
2224         for (i = 0; i < vi->max_queue_pairs; i++) {
2225                 struct virtqueue *vq = vi->sq[i].vq;
2226                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2227                         if (!is_xdp_raw_buffer_queue(vi, i))
2228                                 dev_kfree_skb(buf);
2229                         else
2230                                 put_page(virt_to_head_page(buf));
2231                 }
2232         }
2233
2234         for (i = 0; i < vi->max_queue_pairs; i++) {
2235                 struct virtqueue *vq = vi->rq[i].vq;
2236
2237                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2238                         if (vi->mergeable_rx_bufs) {
2239                                 put_page(virt_to_head_page(buf));
2240                         } else if (vi->big_packets) {
2241                                 give_pages(&vi->rq[i], buf);
2242                         } else {
2243                                 put_page(virt_to_head_page(buf));
2244                         }
2245                 }
2246         }
2247 }
2248
2249 static void virtnet_del_vqs(struct virtnet_info *vi)
2250 {
2251         struct virtio_device *vdev = vi->vdev;
2252
2253         virtnet_clean_affinity(vi, -1);
2254
2255         vdev->config->del_vqs(vdev);
2256
2257         virtnet_free_queues(vi);
2258 }
2259
2260 /* How large should a single buffer be so a queue full of these can fit at
2261  * least one full packet?
2262  * Logic below assumes the mergeable buffer header is used.
2263  */
2264 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2265 {
2266         const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2267         unsigned int rq_size = virtqueue_get_vring_size(vq);
2268         unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2269         unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2270         unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2271
2272         return max(max(min_buf_len, hdr_len) - hdr_len,
2273                    (unsigned int)GOOD_PACKET_LEN);
2274 }
2275
2276 static int virtnet_find_vqs(struct virtnet_info *vi)
2277 {
2278         vq_callback_t **callbacks;
2279         struct virtqueue **vqs;
2280         int ret = -ENOMEM;
2281         int i, total_vqs;
2282         const char **names;
2283         bool *ctx;
2284
2285         /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2286          * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2287          * possible control vq.
2288          */
2289         total_vqs = vi->max_queue_pairs * 2 +
2290                     virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2291
2292         /* Allocate space for find_vqs parameters */
2293         vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
2294         if (!vqs)
2295                 goto err_vq;
2296         callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
2297         if (!callbacks)
2298                 goto err_callback;
2299         names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
2300         if (!names)
2301                 goto err_names;
2302         if (!vi->big_packets || vi->mergeable_rx_bufs) {
2303                 ctx = kzalloc(total_vqs * sizeof(*ctx), GFP_KERNEL);
2304                 if (!ctx)
2305                         goto err_ctx;
2306         } else {
2307                 ctx = NULL;
2308         }
2309
2310         /* Parameters for control virtqueue, if any */
2311         if (vi->has_cvq) {
2312                 callbacks[total_vqs - 1] = NULL;
2313                 names[total_vqs - 1] = "control";
2314         }
2315
2316         /* Allocate/initialize parameters for send/receive virtqueues */
2317         for (i = 0; i < vi->max_queue_pairs; i++) {
2318                 callbacks[rxq2vq(i)] = skb_recv_done;
2319                 callbacks[txq2vq(i)] = skb_xmit_done;
2320                 sprintf(vi->rq[i].name, "input.%d", i);
2321                 sprintf(vi->sq[i].name, "output.%d", i);
2322                 names[rxq2vq(i)] = vi->rq[i].name;
2323                 names[txq2vq(i)] = vi->sq[i].name;
2324                 if (ctx)
2325                         ctx[rxq2vq(i)] = true;
2326         }
2327
2328         ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
2329                                          names, ctx, NULL);
2330         if (ret)
2331                 goto err_find;
2332
2333         if (vi->has_cvq) {
2334                 vi->cvq = vqs[total_vqs - 1];
2335                 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2336                         vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2337         }
2338
2339         for (i = 0; i < vi->max_queue_pairs; i++) {
2340                 vi->rq[i].vq = vqs[rxq2vq(i)];
2341                 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
2342                 vi->sq[i].vq = vqs[txq2vq(i)];
2343         }
2344
2345         kfree(names);
2346         kfree(callbacks);
2347         kfree(vqs);
2348         kfree(ctx);
2349
2350         return 0;
2351
2352 err_find:
2353         kfree(ctx);
2354 err_ctx:
2355         kfree(names);
2356 err_names:
2357         kfree(callbacks);
2358 err_callback:
2359         kfree(vqs);
2360 err_vq:
2361         return ret;
2362 }
2363
2364 static int virtnet_alloc_queues(struct virtnet_info *vi)
2365 {
2366         int i;
2367
2368         vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
2369         if (!vi->ctrl)
2370                 goto err_ctrl;
2371         vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
2372         if (!vi->sq)
2373                 goto err_sq;
2374         vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
2375         if (!vi->rq)
2376                 goto err_rq;
2377
2378         INIT_DELAYED_WORK(&vi->refill, refill_work);
2379         for (i = 0; i < vi->max_queue_pairs; i++) {
2380                 vi->rq[i].pages = NULL;
2381                 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2382                                napi_weight);
2383                 netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2384                                   napi_tx ? napi_weight : 0);
2385
2386                 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
2387                 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
2388                 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
2389         }
2390
2391         return 0;
2392
2393 err_rq:
2394         kfree(vi->sq);
2395 err_sq:
2396         kfree(vi->ctrl);
2397 err_ctrl:
2398         return -ENOMEM;
2399 }
2400
2401 static int init_vqs(struct virtnet_info *vi)
2402 {
2403         int ret;
2404
2405         /* Allocate send & receive queues */
2406         ret = virtnet_alloc_queues(vi);
2407         if (ret)
2408                 goto err;
2409
2410         ret = virtnet_find_vqs(vi);
2411         if (ret)
2412                 goto err_free;
2413
2414         get_online_cpus();
2415         virtnet_set_affinity(vi);
2416         put_online_cpus();
2417
2418         return 0;
2419
2420 err_free:
2421         virtnet_free_queues(vi);
2422 err:
2423         return ret;
2424 }
2425
2426 #ifdef CONFIG_SYSFS
2427 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
2428                 char *buf)
2429 {
2430         struct virtnet_info *vi = netdev_priv(queue->dev);
2431         unsigned int queue_index = get_netdev_rx_queue_index(queue);
2432         struct ewma_pkt_len *avg;
2433
2434         BUG_ON(queue_index >= vi->max_queue_pairs);
2435         avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2436         return sprintf(buf, "%u\n",
2437                        get_mergeable_buf_len(&vi->rq[queue_index], avg));
2438 }
2439
2440 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2441         __ATTR_RO(mergeable_rx_buffer_size);
2442
2443 static struct attribute *virtio_net_mrg_rx_attrs[] = {
2444         &mergeable_rx_buffer_size_attribute.attr,
2445         NULL
2446 };
2447
2448 static const struct attribute_group virtio_net_mrg_rx_group = {
2449         .name = "virtio_net",
2450         .attrs = virtio_net_mrg_rx_attrs
2451 };
2452 #endif
2453
2454 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
2455                                     unsigned int fbit,
2456                                     const char *fname, const char *dname)
2457 {
2458         if (!virtio_has_feature(vdev, fbit))
2459                 return false;
2460
2461         dev_err(&vdev->dev, "device advertises feature %s but not %s",
2462                 fname, dname);
2463
2464         return true;
2465 }
2466
2467 #define VIRTNET_FAIL_ON(vdev, fbit, dbit)                       \
2468         virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2469
2470 static bool virtnet_validate_features(struct virtio_device *vdev)
2471 {
2472         if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
2473             (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
2474                              "VIRTIO_NET_F_CTRL_VQ") ||
2475              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
2476                              "VIRTIO_NET_F_CTRL_VQ") ||
2477              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
2478                              "VIRTIO_NET_F_CTRL_VQ") ||
2479              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
2480              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
2481                              "VIRTIO_NET_F_CTRL_VQ"))) {
2482                 return false;
2483         }
2484
2485         return true;
2486 }
2487
2488 #define MIN_MTU ETH_MIN_MTU
2489 #define MAX_MTU ETH_MAX_MTU
2490
2491 static int virtnet_validate(struct virtio_device *vdev)
2492 {
2493         if (!vdev->config->get) {
2494                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2495                         __func__);
2496                 return -EINVAL;
2497         }
2498
2499         if (!virtnet_validate_features(vdev))
2500                 return -EINVAL;
2501
2502         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2503                 int mtu = virtio_cread16(vdev,
2504                                          offsetof(struct virtio_net_config,
2505                                                   mtu));
2506                 if (mtu < MIN_MTU)
2507                         __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
2508         }
2509
2510         return 0;
2511 }
2512
2513 static int virtnet_probe(struct virtio_device *vdev)
2514 {
2515         int i, err;
2516         struct net_device *dev;
2517         struct virtnet_info *vi;
2518         u16 max_queue_pairs;
2519         int mtu;
2520
2521         /* Find if host supports multiqueue virtio_net device */
2522         err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
2523                                    struct virtio_net_config,
2524                                    max_virtqueue_pairs, &max_queue_pairs);
2525
2526         /* We need at least 2 queue's */
2527         if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
2528             max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
2529             !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2530                 max_queue_pairs = 1;
2531
2532         /* Allocate ourselves a network device with room for our info */
2533         dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
2534         if (!dev)
2535                 return -ENOMEM;
2536
2537         /* Set up network device as normal. */
2538         dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
2539         dev->netdev_ops = &virtnet_netdev;
2540         dev->features = NETIF_F_HIGHDMA;
2541
2542         dev->ethtool_ops = &virtnet_ethtool_ops;
2543         SET_NETDEV_DEV(dev, &vdev->dev);
2544
2545         /* Do we support "hardware" checksums? */
2546         if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
2547                 /* This opens up the world of extra features. */
2548                 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2549                 if (csum)
2550                         dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2551
2552                 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
2553                         dev->hw_features |= NETIF_F_TSO
2554                                 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
2555                 }
2556                 /* Individual feature bits: what can host handle? */
2557                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
2558                         dev->hw_features |= NETIF_F_TSO;
2559                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
2560                         dev->hw_features |= NETIF_F_TSO6;
2561                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
2562                         dev->hw_features |= NETIF_F_TSO_ECN;
2563
2564                 dev->features |= NETIF_F_GSO_ROBUST;
2565
2566                 if (gso)
2567                         dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
2568                 /* (!csum && gso) case will be fixed by register_netdev() */
2569         }
2570         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
2571                 dev->features |= NETIF_F_RXCSUM;
2572
2573         dev->vlan_features = dev->features;
2574
2575         /* MTU range: 68 - 65535 */
2576         dev->min_mtu = MIN_MTU;
2577         dev->max_mtu = MAX_MTU;
2578
2579         /* Configuration may specify what MAC to use.  Otherwise random. */
2580         if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
2581                 virtio_cread_bytes(vdev,
2582                                    offsetof(struct virtio_net_config, mac),
2583                                    dev->dev_addr, dev->addr_len);
2584         else
2585                 eth_hw_addr_random(dev);
2586
2587         /* Set up our device-specific information */
2588         vi = netdev_priv(dev);
2589         vi->dev = dev;
2590         vi->vdev = vdev;
2591         vdev->priv = vi;
2592         vi->stats = alloc_percpu(struct virtnet_stats);
2593         err = -ENOMEM;
2594         if (vi->stats == NULL)
2595                 goto free;
2596
2597         for_each_possible_cpu(i) {
2598                 struct virtnet_stats *virtnet_stats;
2599                 virtnet_stats = per_cpu_ptr(vi->stats, i);
2600                 u64_stats_init(&virtnet_stats->tx_syncp);
2601                 u64_stats_init(&virtnet_stats->rx_syncp);
2602         }
2603
2604         INIT_WORK(&vi->config_work, virtnet_config_changed_work);
2605
2606         /* If we can receive ANY GSO packets, we must allocate large ones. */
2607         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2608             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2609             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
2610             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
2611                 vi->big_packets = true;
2612
2613         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
2614                 vi->mergeable_rx_bufs = true;
2615
2616         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
2617             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2618                 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2619         else
2620                 vi->hdr_len = sizeof(struct virtio_net_hdr);
2621
2622         if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
2623             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2624                 vi->any_header_sg = true;
2625
2626         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2627                 vi->has_cvq = true;
2628
2629         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2630                 mtu = virtio_cread16(vdev,
2631                                      offsetof(struct virtio_net_config,
2632                                               mtu));
2633                 if (mtu < dev->min_mtu) {
2634                         /* Should never trigger: MTU was previously validated
2635                          * in virtnet_validate.
2636                          */
2637                         dev_err(&vdev->dev, "device MTU appears to have changed "
2638                                 "it is now %d < %d", mtu, dev->min_mtu);
2639                         goto free_stats;
2640                 }
2641
2642                 dev->mtu = mtu;
2643                 dev->max_mtu = mtu;
2644
2645                 /* TODO: size buffers correctly in this case. */
2646                 if (dev->mtu > ETH_DATA_LEN)
2647                         vi->big_packets = true;
2648         }
2649
2650         if (vi->any_header_sg)
2651                 dev->needed_headroom = vi->hdr_len;
2652
2653         /* Enable multiqueue by default */
2654         if (num_online_cpus() >= max_queue_pairs)
2655                 vi->curr_queue_pairs = max_queue_pairs;
2656         else
2657                 vi->curr_queue_pairs = num_online_cpus();
2658         vi->max_queue_pairs = max_queue_pairs;
2659
2660         /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
2661         err = init_vqs(vi);
2662         if (err)
2663                 goto free_stats;
2664
2665 #ifdef CONFIG_SYSFS
2666         if (vi->mergeable_rx_bufs)
2667                 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
2668 #endif
2669         netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
2670         netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
2671
2672         virtnet_init_settings(dev);
2673
2674         err = register_netdev(dev);
2675         if (err) {
2676                 pr_debug("virtio_net: registering device failed\n");
2677                 goto free_vqs;
2678         }
2679
2680         virtio_device_ready(vdev);
2681
2682         err = virtnet_cpu_notif_add(vi);
2683         if (err) {
2684                 pr_debug("virtio_net: registering cpu notifier failed\n");
2685                 goto free_unregister_netdev;
2686         }
2687
2688         virtnet_set_queues(vi, vi->curr_queue_pairs);
2689
2690         /* Assume link up if device can't report link status,
2691            otherwise get link status from config. */
2692         netif_carrier_off(dev);
2693         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2694                 schedule_work(&vi->config_work);
2695         } else {
2696                 vi->status = VIRTIO_NET_S_LINK_UP;
2697                 netif_carrier_on(dev);
2698         }
2699
2700         for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
2701                 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
2702                         set_bit(guest_offloads[i], &vi->guest_offloads);
2703
2704         pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
2705                  dev->name, max_queue_pairs);
2706
2707         return 0;
2708
2709 free_unregister_netdev:
2710         vi->vdev->config->reset(vdev);
2711
2712         unregister_netdev(dev);
2713 free_vqs:
2714         cancel_delayed_work_sync(&vi->refill);
2715         free_receive_page_frags(vi);
2716         virtnet_del_vqs(vi);
2717 free_stats:
2718         free_percpu(vi->stats);
2719 free:
2720         free_netdev(dev);
2721         return err;
2722 }
2723
2724 static void remove_vq_common(struct virtnet_info *vi)
2725 {
2726         vi->vdev->config->reset(vi->vdev);
2727
2728         /* Free unused buffers in both send and recv, if any. */
2729         free_unused_bufs(vi);
2730
2731         free_receive_bufs(vi);
2732
2733         free_receive_page_frags(vi);
2734
2735         virtnet_del_vqs(vi);
2736 }
2737
2738 static void virtnet_remove(struct virtio_device *vdev)
2739 {
2740         struct virtnet_info *vi = vdev->priv;
2741
2742         virtnet_cpu_notif_remove(vi);
2743
2744         /* Make sure no work handler is accessing the device. */
2745         flush_work(&vi->config_work);
2746
2747         unregister_netdev(vi->dev);
2748
2749         remove_vq_common(vi);
2750
2751         free_percpu(vi->stats);
2752         free_netdev(vi->dev);
2753 }
2754
2755 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
2756 {
2757         struct virtnet_info *vi = vdev->priv;
2758
2759         virtnet_cpu_notif_remove(vi);
2760         virtnet_freeze_down(vdev);
2761         remove_vq_common(vi);
2762
2763         return 0;
2764 }
2765
2766 static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
2767 {
2768         struct virtnet_info *vi = vdev->priv;
2769         int err;
2770
2771         err = virtnet_restore_up(vdev);
2772         if (err)
2773                 return err;
2774         virtnet_set_queues(vi, vi->curr_queue_pairs);
2775
2776         err = virtnet_cpu_notif_add(vi);
2777         if (err) {
2778                 virtnet_freeze_down(vdev);
2779                 remove_vq_common(vi);
2780                 return err;
2781         }
2782
2783         return 0;
2784 }
2785
2786 static struct virtio_device_id id_table[] = {
2787         { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
2788         { 0 },
2789 };
2790
2791 #define VIRTNET_FEATURES \
2792         VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
2793         VIRTIO_NET_F_MAC, \
2794         VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
2795         VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
2796         VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
2797         VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
2798         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
2799         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
2800         VIRTIO_NET_F_CTRL_MAC_ADDR, \
2801         VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
2802
2803 static unsigned int features[] = {
2804         VIRTNET_FEATURES,
2805 };
2806
2807 static unsigned int features_legacy[] = {
2808         VIRTNET_FEATURES,
2809         VIRTIO_NET_F_GSO,
2810         VIRTIO_F_ANY_LAYOUT,
2811 };
2812
2813 static struct virtio_driver virtio_net_driver = {
2814         .feature_table = features,
2815         .feature_table_size = ARRAY_SIZE(features),
2816         .feature_table_legacy = features_legacy,
2817         .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
2818         .driver.name =  KBUILD_MODNAME,
2819         .driver.owner = THIS_MODULE,
2820         .id_table =     id_table,
2821         .validate =     virtnet_validate,
2822         .probe =        virtnet_probe,
2823         .remove =       virtnet_remove,
2824         .config_changed = virtnet_config_changed,
2825 #ifdef CONFIG_PM_SLEEP
2826         .freeze =       virtnet_freeze,
2827         .restore =      virtnet_restore,
2828 #endif
2829 };
2830
2831 static __init int virtio_net_driver_init(void)
2832 {
2833         int ret;
2834
2835         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
2836                                       virtnet_cpu_online,
2837                                       virtnet_cpu_down_prep);
2838         if (ret < 0)
2839                 goto out;
2840         virtionet_online = ret;
2841         ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
2842                                       NULL, virtnet_cpu_dead);
2843         if (ret)
2844                 goto err_dead;
2845
2846         ret = register_virtio_driver(&virtio_net_driver);
2847         if (ret)
2848                 goto err_virtio;
2849         return 0;
2850 err_virtio:
2851         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2852 err_dead:
2853         cpuhp_remove_multi_state(virtionet_online);
2854 out:
2855         return ret;
2856 }
2857 module_init(virtio_net_driver_init);
2858
2859 static __exit void virtio_net_driver_exit(void)
2860 {
2861         unregister_virtio_driver(&virtio_net_driver);
2862         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2863         cpuhp_remove_multi_state(virtionet_online);
2864 }
2865 module_exit(virtio_net_driver_exit);
2866
2867 MODULE_DEVICE_TABLE(virtio, id_table);
2868 MODULE_DESCRIPTION("Virtio network driver");
2869 MODULE_LICENSE("GPL");