drivers/virtio/virtio_ring.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* Virtio ring implementation.
   3  *
   4  *  Copyright 2007 Rusty Russell IBM Corporation
   5  */
   6 #include <linux/virtio.h>
   7 #include <linux/virtio_ring.h>
   8 #include <linux/virtio_config.h>
   9 #include <linux/device.h>
  10 #include <linux/slab.h>
  11 #include <linux/module.h>
  12 #include <linux/hrtimer.h>
  13 #include <linux/dma-mapping.h>
  14 #include <xen/xen.h>
  15
  16 #ifdef DEBUG
  17 /* For development, we want to crash whenever the ring is screwed. */
  18 #define BAD_RING(_vq, fmt, args...)                             \
  19         do {                                                    \
  20                 dev_err(&(_vq)->vq.vdev->dev,                   \
  21                         "%s:"fmt, (_vq)->vq.name, ##args);      \
  22                 BUG();                                          \
  23         } while (0)
  24 /* Caller is supposed to guarantee no reentry. */
  25 #define START_USE(_vq)                                          \
  26         do {                                                    \
  27                 if ((_vq)->in_use)                              \
  28                         panic("%s:in_use = %i\n",               \
  29                               (_vq)->vq.name, (_vq)->in_use);   \
  30                 (_vq)->in_use = __LINE__;                       \
  31         } while (0)
  32 #define END_USE(_vq) \
  33         do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
  34 #define LAST_ADD_TIME_UPDATE(_vq)                               \
  35         do {                                                    \
  36                 ktime_t now = ktime_get();                      \
  37                                                                 \
  38                 /* No kick or get, with .1 second between?  Warn. */ \
  39                 if ((_vq)->last_add_time_valid)                 \
  40                         WARN_ON(ktime_to_ms(ktime_sub(now,      \
  41                                 (_vq)->last_add_time)) > 100);  \
  42                 (_vq)->last_add_time = now;                     \
  43                 (_vq)->last_add_time_valid = true;              \
  44         } while (0)
  45 #define LAST_ADD_TIME_CHECK(_vq)                                \
  46         do {                                                    \
  47                 if ((_vq)->last_add_time_valid) {               \
  48                         WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
  49                                       (_vq)->last_add_time)) > 100); \
  50                 }                                               \
  51         } while (0)
  52 #define LAST_ADD_TIME_INVALID(_vq)                              \
  53         ((_vq)->last_add_time_valid = false)
  54 #else
  55 #define BAD_RING(_vq, fmt, args...)                             \
  56         do {                                                    \
  57                 dev_err(&_vq->vq.vdev->dev,                     \
  58                         "%s:"fmt, (_vq)->vq.name, ##args);      \
  59                 (_vq)->broken = true;                           \
  60         } while (0)
  61 #define START_USE(vq)
  62 #define END_USE(vq)
  63 #define LAST_ADD_TIME_UPDATE(vq)
  64 #define LAST_ADD_TIME_CHECK(vq)
  65 #define LAST_ADD_TIME_INVALID(vq)
  66 #endif
  67
  68 struct vring_desc_state_split {
  69         void *data;                     /* Data for callback. */
  70         struct vring_desc *indir_desc;  /* Indirect descriptor, if any. */
  71 };
  72
  73 struct vring_desc_state_packed {
  74         void *data;                     /* Data for callback. */
  75         struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
  76         u16 num;                        /* Descriptor list length. */
  77         u16 next;                       /* The next desc state in a list. */
  78         u16 last;                       /* The last desc state in a list. */
  79 };
  80
  81 struct vring_desc_extra_packed {
  82         dma_addr_t addr;                /* Buffer DMA addr. */
  83         u32 len;                        /* Buffer length. */
  84         u16 flags;                      /* Descriptor flags. */
  85 };
  86
  87 struct vring_virtqueue {
  88         struct virtqueue vq;
  89
  90         /* Is this a packed ring? */
  91         bool packed_ring;
  92
  93         /* Is DMA API used? */
  94         bool use_dma_api;
  95
  96         /* Can we use weak barriers? */
  97         bool weak_barriers;
  98
  99         /* Other side has made a mess, don't try any more. */
 100         bool broken;
 101
 102         /* Host supports indirect buffers */
 103         bool indirect;
 104
 105         /* Host publishes avail event idx */
 106         bool event;
 107
 108         /* Head of free buffer list. */
 109         unsigned int free_head;
 110         /* Number we've added since last sync. */
 111         unsigned int num_added;
 112
 113         /* Last used index we've seen. */
 114         u16 last_used_idx;
 115
 116         union {
 117                 /* Available for split ring */
 118                 struct {
 119                         /* Actual memory layout for this queue. */
 120                         struct vring vring;
 121
 122                         /* Last written value to avail->flags */
 123                         u16 avail_flags_shadow;
 124
 125                         /*
 126                          * Last written value to avail->idx in
 127                          * guest byte order.
 128                          */
 129                         u16 avail_idx_shadow;
 130
 131                         /* Per-descriptor state. */
 132                         struct vring_desc_state_split *desc_state;
 133
 134                         /* DMA address and size information */
 135                         dma_addr_t queue_dma_addr;
 136                         size_t queue_size_in_bytes;
 137                 } split;
 138
 139                 /* Available for packed ring */
 140                 struct {
 141                         /* Actual memory layout for this queue. */
 142                         struct {
 143                                 unsigned int num;
 144                                 struct vring_packed_desc *desc;
 145                                 struct vring_packed_desc_event *driver;
 146                                 struct vring_packed_desc_event *device;
 147                         } vring;
 148
 149                         /* Driver ring wrap counter. */
 150                         bool avail_wrap_counter;
 151
 152                         /* Device ring wrap counter. */
 153                         bool used_wrap_counter;
 154
 155                         /* Avail used flags. */
 156                         u16 avail_used_flags;
 157
 158                         /* Index of the next avail descriptor. */
 159                         u16 next_avail_idx;
 160
 161                         /*
 162                          * Last written value to driver->flags in
 163                          * guest byte order.
 164                          */
 165                         u16 event_flags_shadow;
 166
 167                         /* Per-descriptor state. */
 168                         struct vring_desc_state_packed *desc_state;
 169                         struct vring_desc_extra_packed *desc_extra;
 170
 171                         /* DMA address and size information */
 172                         dma_addr_t ring_dma_addr;
 173                         dma_addr_t driver_event_dma_addr;
 174                         dma_addr_t device_event_dma_addr;
 175                         size_t ring_size_in_bytes;
 176                         size_t event_size_in_bytes;
 177                 } packed;
 178         };
 179
 180         /* How to notify other side. FIXME: commonalize hcalls! */
 181         bool (*notify)(struct virtqueue *vq);
 182
 183         /* DMA, allocation, and size information */
 184         bool we_own_ring;
 185
 186 #ifdef DEBUG
 187         /* They're supposed to lock for us. */
 188         unsigned int in_use;
 189
 190         /* Figure out if their kicks are too delayed. */
 191         bool last_add_time_valid;
 192         ktime_t last_add_time;
 193 #endif
 194 };
 195
 196
 197 /*
 198  * Helpers.
 199  */
 200
 201 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 202
 203 static inline bool virtqueue_use_indirect(struct virtqueue *_vq,
 204                                           unsigned int total_sg)
 205 {
 206         struct vring_virtqueue *vq = to_vvq(_vq);
 207
 208         /*
 209          * If the host supports indirect descriptor tables, and we have multiple
 210          * buffers, then go indirect. FIXME: tune this threshold
 211          */
 212         return (vq->indirect && total_sg > 1 && vq->vq.num_free);
 213 }
 214
 215 /*
 216  * Modern virtio devices have feature bits to specify whether they need a
 217  * quirk and bypass the IOMMU. If not there, just use the DMA API.
 218  *
 219  * If there, the interaction between virtio and DMA API is messy.
 220  *
 221  * On most systems with virtio, physical addresses match bus addresses,
 222  * and it doesn't particularly matter whether we use the DMA API.
 223  *
 224  * On some systems, including Xen and any system with a physical device
 225  * that speaks virtio behind a physical IOMMU, we must use the DMA API
 226  * for virtio DMA to work at all.
 227  *
 228  * On other systems, including SPARC and PPC64, virtio-pci devices are
 229  * enumerated as though they are behind an IOMMU, but the virtio host
 230  * ignores the IOMMU, so we must either pretend that the IOMMU isn't
 231  * there or somehow map everything as the identity.
 232  *
 233  * For the time being, we preserve historic behavior and bypass the DMA
 234  * API.
 235  *
 236  * TODO: install a per-device DMA ops structure that does the right thing
 237  * taking into account all the above quirks, and use the DMA API
 238  * unconditionally on data path.
 239  */
 240
 241 static bool vring_use_dma_api(struct virtio_device *vdev)
 242 {
 243         if (!virtio_has_dma_quirk(vdev))
 244                 return true;
 245
 246         /* Otherwise, we are left to guess. */
 247         /*
 248          * In theory, it's possible to have a buggy QEMU-supposed
 249          * emulated Q35 IOMMU and Xen enabled at the same time.  On
 250          * such a configuration, virtio has never worked and will
 251          * not work without an even larger kludge.  Instead, enable
 252          * the DMA API if we're a Xen guest, which at least allows
 253          * all of the sensible Xen configurations to work correctly.
 254          */
 255         if (xen_domain())
 256                 return true;
 257
 258         return false;
 259 }
 260
 261 size_t virtio_max_dma_size(struct virtio_device *vdev)
 262 {
 263         size_t max_segment_size = SIZE_MAX;
 264
 265         if (vring_use_dma_api(vdev))
 266                 max_segment_size = dma_max_mapping_size(vdev->dev.parent);
 267
 268         return max_segment_size;
 269 }
 270 EXPORT_SYMBOL_GPL(virtio_max_dma_size);
 271
 272 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
 273                               dma_addr_t *dma_handle, gfp_t flag)
 274 {
 275         if (vring_use_dma_api(vdev)) {
 276                 return dma_alloc_coherent(vdev->dev.parent, size,
 277                                           dma_handle, flag);
 278         } else {
 279                 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
 280
 281                 if (queue) {
 282                         phys_addr_t phys_addr = virt_to_phys(queue);
 283                         *dma_handle = (dma_addr_t)phys_addr;
 284
 285                         /*
 286                          * Sanity check: make sure we dind't truncate
 287                          * the address.  The only arches I can find that
 288                          * have 64-bit phys_addr_t but 32-bit dma_addr_t
 289                          * are certain non-highmem MIPS and x86
 290                          * configurations, but these configurations
 291                          * should never allocate physical pages above 32
 292                          * bits, so this is fine.  Just in case, throw a
 293                          * warning and abort if we end up with an
 294                          * unrepresentable address.
 295                          */
 296                         if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
 297                                 free_pages_exact(queue, PAGE_ALIGN(size));
 298                                 return NULL;
 299                         }
 300                 }
 301                 return queue;
 302         }
 303 }
 304
 305 static void vring_free_queue(struct virtio_device *vdev, size_t size,
 306                              void *queue, dma_addr_t dma_handle)
 307 {
 308         if (vring_use_dma_api(vdev))
 309                 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
 310         else
 311                 free_pages_exact(queue, PAGE_ALIGN(size));
 312 }
 313
 314 /*
 315  * The DMA ops on various arches are rather gnarly right now, and
 316  * making all of the arch DMA ops work on the vring device itself
 317  * is a mess.  For now, we use the parent device for DMA ops.
 318  */
 319 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 320 {
 321         return vq->vq.vdev->dev.parent;
 322 }
 323
 324 /* Map one sg entry. */
 325 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
 326                                    struct scatterlist *sg,
 327                                    enum dma_data_direction direction)
 328 {
 329         if (!vq->use_dma_api)
 330                 return (dma_addr_t)sg_phys(sg);
 331
 332         /*
 333          * We can't use dma_map_sg, because we don't use scatterlists in
 334          * the way it expects (we don't guarantee that the scatterlist
 335          * will exist for the lifetime of the mapping).
 336          */
 337         return dma_map_page(vring_dma_dev(vq),
 338                             sg_page(sg), sg->offset, sg->length,
 339                             direction);
 340 }
 341
 342 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
 343                                    void *cpu_addr, size_t size,
 344                                    enum dma_data_direction direction)
 345 {
 346         if (!vq->use_dma_api)
 347                 return (dma_addr_t)virt_to_phys(cpu_addr);
 348
 349         return dma_map_single(vring_dma_dev(vq),
 350                               cpu_addr, size, direction);
 351 }
 352
 353 static int vring_mapping_error(const struct vring_virtqueue *vq,
 354                                dma_addr_t addr)
 355 {
 356         if (!vq->use_dma_api)
 357                 return 0;
 358
 359         return dma_mapping_error(vring_dma_dev(vq), addr);
 360 }
 361
 362
 363 /*
 364  * Split ring specific functions - *_split().
 365  */
 366
 367 static void vring_unmap_one_split(const struct vring_virtqueue *vq,
 368                                   struct vring_desc *desc)
 369 {
 370         u16 flags;
 371
 372         if (!vq->use_dma_api)
 373                 return;
 374
 375         flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
 376
 377         if (flags & VRING_DESC_F_INDIRECT) {
 378                 dma_unmap_single(vring_dma_dev(vq),
 379                                  virtio64_to_cpu(vq->vq.vdev, desc->addr),
 380                                  virtio32_to_cpu(vq->vq.vdev, desc->len),
 381                                  (flags & VRING_DESC_F_WRITE) ?
 382                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 383         } else {
 384                 dma_unmap_page(vring_dma_dev(vq),
 385                                virtio64_to_cpu(vq->vq.vdev, desc->addr),
 386                                virtio32_to_cpu(vq->vq.vdev, desc->len),
 387                                (flags & VRING_DESC_F_WRITE) ?
 388                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 389         }
 390 }
 391
 392 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
 393                                                unsigned int total_sg,
 394                                                gfp_t gfp)
 395 {
 396         struct vring_desc *desc;
 397         unsigned int i;
 398
 399         /*
 400          * We require lowmem mappings for the descriptors because
 401          * otherwise virt_to_phys will give us bogus addresses in the
 402          * virtqueue.
 403          */
 404         gfp &= ~__GFP_HIGHMEM;
 405
 406         desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
 407         if (!desc)
 408                 return NULL;
 409
 410         for (i = 0; i < total_sg; i++)
 411                 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
 412         return desc;
 413 }
 414
 415 static inline int virtqueue_add_split(struct virtqueue *_vq,
 416                                       struct scatterlist *sgs[],
 417                                       unsigned int total_sg,
 418                                       unsigned int out_sgs,
 419                                       unsigned int in_sgs,
 420                                       void *data,
 421                                       void *ctx,
 422                                       gfp_t gfp)
 423 {
 424         struct vring_virtqueue *vq = to_vvq(_vq);
 425         struct scatterlist *sg;
 426         struct vring_desc *desc;
 427         unsigned int i, n, avail, descs_used, prev, err_idx;
 428         int head;
 429         bool indirect;
 430
 431         START_USE(vq);
 432
 433         BUG_ON(data == NULL);
 434         BUG_ON(ctx && vq->indirect);
 435
 436         if (unlikely(vq->broken)) {
 437                 END_USE(vq);
 438                 return -EIO;
 439         }
 440
 441         LAST_ADD_TIME_UPDATE(vq);
 442
 443         BUG_ON(total_sg == 0);
 444
 445         head = vq->free_head;
 446
 447         if (virtqueue_use_indirect(_vq, total_sg))
 448                 desc = alloc_indirect_split(_vq, total_sg, gfp);
 449         else {
 450                 desc = NULL;
 451                 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
 452         }
 453
 454         if (desc) {
 455                 /* Use a single buffer which doesn't continue */
 456                 indirect = true;
 457                 /* Set up rest to use this indirect table. */
 458                 i = 0;
 459                 descs_used = 1;
 460         } else {
 461                 indirect = false;
 462                 desc = vq->split.vring.desc;
 463                 i = head;
 464                 descs_used = total_sg;
 465         }
 466
 467         if (vq->vq.num_free < descs_used) {
 468                 pr_debug("Can't add buf len %i - avail = %i\n",
 469                          descs_used, vq->vq.num_free);
 470                 /* FIXME: for historical reasons, we force a notify here if
 471                  * there are outgoing parts to the buffer.  Presumably the
 472                  * host should service the ring ASAP. */
 473                 if (out_sgs)
 474                         vq->notify(&vq->vq);
 475                 if (indirect)
 476                         kfree(desc);
 477                 END_USE(vq);
 478                 return -ENOSPC;
 479         }
 480
 481         for (n = 0; n < out_sgs; n++) {
 482                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
 483                         dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
 484                         if (vring_mapping_error(vq, addr))
 485                                 goto unmap_release;
 486
 487                         desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
 488                         desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
 489                         desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
 490                         prev = i;
 491                         i = virtio16_to_cpu(_vq->vdev, desc[i].next);
 492                 }
 493         }
 494         for (; n < (out_sgs + in_sgs); n++) {
 495                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
 496                         dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
 497                         if (vring_mapping_error(vq, addr))
 498                                 goto unmap_release;
 499
 500                         desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
 501                         desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
 502                         desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
 503                         prev = i;
 504                         i = virtio16_to_cpu(_vq->vdev, desc[i].next);
 505                 }
 506         }
 507         /* Last one doesn't continue. */
 508         desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
 509
 510         if (indirect) {
 511                 /* Now that the indirect table is filled in, map it. */
 512                 dma_addr_t addr = vring_map_single(
 513                         vq, desc, total_sg * sizeof(struct vring_desc),
 514                         DMA_TO_DEVICE);
 515                 if (vring_mapping_error(vq, addr))
 516                         goto unmap_release;
 517
 518                 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
 519                                 VRING_DESC_F_INDIRECT);
 520                 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
 521                                 addr);
 522
 523                 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
 524                                 total_sg * sizeof(struct vring_desc));
 525         }
 526
 527         /* We're using some buffers from the free list. */
 528         vq->vq.num_free -= descs_used;
 529
 530         /* Update free pointer */
 531         if (indirect)
 532                 vq->free_head = virtio16_to_cpu(_vq->vdev,
 533                                         vq->split.vring.desc[head].next);
 534         else
 535                 vq->free_head = i;
 536
 537         /* Store token and indirect buffer state. */
 538         vq->split.desc_state[head].data = data;
 539         if (indirect)
 540                 vq->split.desc_state[head].indir_desc = desc;
 541         else
 542                 vq->split.desc_state[head].indir_desc = ctx;
 543
 544         /* Put entry in available array (but don't update avail->idx until they
 545          * do sync). */
 546         avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
 547         vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
 548
 549         /* Descriptors and available array need to be set before we expose the
 550          * new available array entries. */
 551         virtio_wmb(vq->weak_barriers);
 552         vq->split.avail_idx_shadow++;
 553         vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
 554                                                 vq->split.avail_idx_shadow);
 555         vq->num_added++;
 556
 557         pr_debug("Added buffer head %i to %p\n", head, vq);
 558         END_USE(vq);
 559
 560         /* This is very unlikely, but theoretically possible.  Kick
 561          * just in case. */
 562         if (unlikely(vq->num_added == (1 << 16) - 1))
 563                 virtqueue_kick(_vq);
 564
 565         return 0;
 566
 567 unmap_release:
 568         err_idx = i;
 569
 570         if (indirect)
 571                 i = 0;
 572         else
 573                 i = head;
 574
 575         for (n = 0; n < total_sg; n++) {
 576                 if (i == err_idx)
 577                         break;
 578                 vring_unmap_one_split(vq, &desc[i]);
 579                 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
 580         }
 581
 582         if (indirect)
 583                 kfree(desc);
 584
 585         END_USE(vq);
 586         return -ENOMEM;
 587 }
 588
 589 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
 590 {
 591         struct vring_virtqueue *vq = to_vvq(_vq);
 592         u16 new, old;
 593         bool needs_kick;
 594
 595         START_USE(vq);
 596         /* We need to expose available array entries before checking avail
 597          * event. */
 598         virtio_mb(vq->weak_barriers);
 599
 600         old = vq->split.avail_idx_shadow - vq->num_added;
 601         new = vq->split.avail_idx_shadow;
 602         vq->num_added = 0;
 603
 604         LAST_ADD_TIME_CHECK(vq);
 605         LAST_ADD_TIME_INVALID(vq);
 606
 607         if (vq->event) {
 608                 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
 609                                         vring_avail_event(&vq->split.vring)),
 610                                               new, old);
 611         } else {
 612                 needs_kick = !(vq->split.vring.used->flags &
 613                                         cpu_to_virtio16(_vq->vdev,
 614                                                 VRING_USED_F_NO_NOTIFY));
 615         }
 616         END_USE(vq);
 617         return needs_kick;
 618 }
 619
 620 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 621                              void **ctx)
 622 {
 623         unsigned int i, j;
 624         __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
 625
 626         /* Clear data ptr. */
 627         vq->split.desc_state[head].data = NULL;
 628
 629         /* Put back on free list: unmap first-level descriptors and find end */
 630         i = head;
 631
 632         while (vq->split.vring.desc[i].flags & nextflag) {
 633                 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
 634                 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
 635                 vq->vq.num_free++;
 636         }
 637
 638         vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
 639         vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
 640                                                 vq->free_head);
 641         vq->free_head = head;
 642
 643         /* Plus final descriptor */
 644         vq->vq.num_free++;
 645
 646         if (vq->indirect) {
 647                 struct vring_desc *indir_desc =
 648                                 vq->split.desc_state[head].indir_desc;
 649                 u32 len;
 650
 651                 /* Free the indirect table, if any, now that it's unmapped. */
 652                 if (!indir_desc)
 653                         return;
 654
 655                 len = virtio32_to_cpu(vq->vq.vdev,
 656                                 vq->split.vring.desc[head].len);
 657
 658                 BUG_ON(!(vq->split.vring.desc[head].flags &
 659                          cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
 660                 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
 661
 662                 for (j = 0; j < len / sizeof(struct vring_desc); j++)
 663                         vring_unmap_one_split(vq, &indir_desc[j]);
 664
 665                 kfree(indir_desc);
 666                 vq->split.desc_state[head].indir_desc = NULL;
 667         } else if (ctx) {
 668                 *ctx = vq->split.desc_state[head].indir_desc;
 669         }
 670 }
 671
 672 static inline bool more_used_split(const struct vring_virtqueue *vq)
 673 {
 674         return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 675                         vq->split.vring.used->idx);
 676 }
 677
 678 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
 679                                          unsigned int *len,
 680                                          void **ctx)
 681 {
 682         struct vring_virtqueue *vq = to_vvq(_vq);
 683         void *ret;
 684         unsigned int i;
 685         u16 last_used;
 686
 687         START_USE(vq);
 688
 689         if (unlikely(vq->broken)) {
 690                 END_USE(vq);
 691                 return NULL;
 692         }
 693
 694         if (!more_used_split(vq)) {
 695                 pr_debug("No more buffers in queue\n");
 696                 END_USE(vq);
 697                 return NULL;
 698         }
 699
 700         /* Only get used array entries after they have been exposed by host. */
 701         virtio_rmb(vq->weak_barriers);
 702
 703         last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
 704         i = virtio32_to_cpu(_vq->vdev,
 705                         vq->split.vring.used->ring[last_used].id);
 706         *len = virtio32_to_cpu(_vq->vdev,
 707                         vq->split.vring.used->ring[last_used].len);
 708
 709         if (unlikely(i >= vq->split.vring.num)) {
 710                 BAD_RING(vq, "id %u out of range\n", i);
 711                 return NULL;
 712         }
 713         if (unlikely(!vq->split.desc_state[i].data)) {
 714                 BAD_RING(vq, "id %u is not a head!\n", i);
 715                 return NULL;
 716         }
 717
 718         /* detach_buf_split clears data, so grab it now. */
 719         ret = vq->split.desc_state[i].data;
 720         detach_buf_split(vq, i, ctx);
 721         vq->last_used_idx++;
 722         /* If we expect an interrupt for the next entry, tell host
 723          * by writing event index and flush out the write before
 724          * the read in the next get_buf call. */
 725         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
 726                 virtio_store_mb(vq->weak_barriers,
 727                                 &vring_used_event(&vq->split.vring),
 728                                 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
 729
 730         LAST_ADD_TIME_INVALID(vq);
 731
 732         END_USE(vq);
 733         return ret;
 734 }
 735
 736 static void virtqueue_disable_cb_split(struct virtqueue *_vq)
 737 {
 738         struct vring_virtqueue *vq = to_vvq(_vq);
 739
 740         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
 741                 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
 742                 if (!vq->event)
 743                         vq->split.vring.avail->flags =
 744                                 cpu_to_virtio16(_vq->vdev,
 745                                                 vq->split.avail_flags_shadow);
 746         }
 747 }
 748
 749 static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
 750 {
 751         struct vring_virtqueue *vq = to_vvq(_vq);
 752         u16 last_used_idx;
 753
 754         START_USE(vq);
 755
 756         /* We optimistically turn back on interrupts, then check if there was
 757          * more to do. */
 758         /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
 759          * either clear the flags bit or point the event index at the next
 760          * entry. Always do both to keep code simple. */
 761         if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
 762                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 763                 if (!vq->event)
 764                         vq->split.vring.avail->flags =
 765                                 cpu_to_virtio16(_vq->vdev,
 766                                                 vq->split.avail_flags_shadow);
 767         }
 768         vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
 769                         last_used_idx = vq->last_used_idx);
 770         END_USE(vq);
 771         return last_used_idx;
 772 }
 773
 774 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
 775 {
 776         struct vring_virtqueue *vq = to_vvq(_vq);
 777
 778         return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
 779                         vq->split.vring.used->idx);
 780 }
 781
 782 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
 783 {
 784         struct vring_virtqueue *vq = to_vvq(_vq);
 785         u16 bufs;
 786
 787         START_USE(vq);
 788
 789         /* We optimistically turn back on interrupts, then check if there was
 790          * more to do. */
 791         /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
 792          * either clear the flags bit or point the event index at the next
 793          * entry. Always update the event index to keep code simple. */
 794         if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
 795                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 796                 if (!vq->event)
 797                         vq->split.vring.avail->flags =
 798                                 cpu_to_virtio16(_vq->vdev,
 799                                                 vq->split.avail_flags_shadow);
 800         }
 801         /* TODO: tune this threshold */
 802         bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
 803
 804         virtio_store_mb(vq->weak_barriers,
 805                         &vring_used_event(&vq->split.vring),
 806                         cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
 807
 808         if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
 809                                         - vq->last_used_idx) > bufs)) {
 810                 END_USE(vq);
 811                 return false;
 812         }
 813
 814         END_USE(vq);
 815         return true;
 816 }
 817
 818 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
 819 {
 820         struct vring_virtqueue *vq = to_vvq(_vq);
 821         unsigned int i;
 822         void *buf;
 823
 824         START_USE(vq);
 825
 826         for (i = 0; i < vq->split.vring.num; i++) {
 827                 if (!vq->split.desc_state[i].data)
 828                         continue;
 829                 /* detach_buf_split clears data, so grab it now. */
 830                 buf = vq->split.desc_state[i].data;
 831                 detach_buf_split(vq, i, NULL);
 832                 vq->split.avail_idx_shadow--;
 833                 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
 834                                 vq->split.avail_idx_shadow);
 835                 END_USE(vq);
 836                 return buf;
 837         }
 838         /* That should have freed everything. */
 839         BUG_ON(vq->vq.num_free != vq->split.vring.num);
 840
 841         END_USE(vq);
 842         return NULL;
 843 }
 844
 845 static struct virtqueue *vring_create_virtqueue_split(
 846         unsigned int index,
 847         unsigned int num,
 848         unsigned int vring_align,
 849         struct virtio_device *vdev,
 850         bool weak_barriers,
 851         bool may_reduce_num,
 852         bool context,
 853         bool (*notify)(struct virtqueue *),
 854         void (*callback)(struct virtqueue *),
 855         const char *name)
 856 {
 857         struct virtqueue *vq;
 858         void *queue = NULL;
 859         dma_addr_t dma_addr;
 860         size_t queue_size_in_bytes;
 861         struct vring vring;
 862
 863         /* We assume num is a power of 2. */
 864         if (num & (num - 1)) {
 865                 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
 866                 return NULL;
 867         }
 868
 869         /* TODO: allocate each queue chunk individually */
 870         for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
 871                 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
 872                                           &dma_addr,
 873                                           GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
 874                 if (queue)
 875                         break;
 876                 if (!may_reduce_num)
 877                         return NULL;
 878         }
 879
 880         if (!num)
 881                 return NULL;
 882
 883         if (!queue) {
 884                 /* Try to get a single page. You are my only hope! */
 885                 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
 886                                           &dma_addr, GFP_KERNEL|__GFP_ZERO);
 887         }
 888         if (!queue)
 889                 return NULL;
 890
 891         queue_size_in_bytes = vring_size(num, vring_align);
 892         vring_init(&vring, num, queue, vring_align);
 893
 894         vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
 895                                    notify, callback, name);
 896         if (!vq) {
 897                 vring_free_queue(vdev, queue_size_in_bytes, queue,
 898                                  dma_addr);
 899                 return NULL;
 900         }
 901
 902         to_vvq(vq)->split.queue_dma_addr = dma_addr;
 903         to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
 904         to_vvq(vq)->we_own_ring = true;
 905
 906         return vq;
 907 }
 908
 909
 910 /*
 911  * Packed ring specific functions - *_packed().
 912  */
 913
 914 static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
 915                                      struct vring_desc_extra_packed *state)
 916 {
 917         u16 flags;
 918
 919         if (!vq->use_dma_api)
 920                 return;
 921
 922         flags = state->flags;
 923
 924         if (flags & VRING_DESC_F_INDIRECT) {
 925                 dma_unmap_single(vring_dma_dev(vq),
 926                                  state->addr, state->len,
 927                                  (flags & VRING_DESC_F_WRITE) ?
 928                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 929         } else {
 930                 dma_unmap_page(vring_dma_dev(vq),
 931                                state->addr, state->len,
 932                                (flags & VRING_DESC_F_WRITE) ?
 933                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 934         }
 935 }
 936
 937 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
 938                                    struct vring_packed_desc *desc)
 939 {
 940         u16 flags;
 941
 942         if (!vq->use_dma_api)
 943                 return;
 944
 945         flags = le16_to_cpu(desc->flags);
 946
 947         if (flags & VRING_DESC_F_INDIRECT) {
 948                 dma_unmap_single(vring_dma_dev(vq),
 949                                  le64_to_cpu(desc->addr),
 950                                  le32_to_cpu(desc->len),
 951                                  (flags & VRING_DESC_F_WRITE) ?
 952                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 953         } else {
 954                 dma_unmap_page(vring_dma_dev(vq),
 955                                le64_to_cpu(desc->addr),
 956                                le32_to_cpu(desc->len),
 957                                (flags & VRING_DESC_F_WRITE) ?
 958                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 959         }
 960 }
 961
 962 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
 963                                                        gfp_t gfp)
 964 {
 965         struct vring_packed_desc *desc;
 966
 967         /*
 968          * We require lowmem mappings for the descriptors because
 969          * otherwise virt_to_phys will give us bogus addresses in the
 970          * virtqueue.
 971          */
 972         gfp &= ~__GFP_HIGHMEM;
 973
 974         desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
 975
 976         return desc;
 977 }
 978
 979 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 980                                        struct scatterlist *sgs[],
 981                                        unsigned int total_sg,
 982                                        unsigned int out_sgs,
 983                                        unsigned int in_sgs,
 984                                        void *data,
 985                                        gfp_t gfp)
 986 {
 987         struct vring_packed_desc *desc;
 988         struct scatterlist *sg;
 989         unsigned int i, n, err_idx;
 990         u16 head, id;
 991         dma_addr_t addr;
 992
 993         head = vq->packed.next_avail_idx;
 994         desc = alloc_indirect_packed(total_sg, gfp);
 995         if (!desc)
 996                 return -ENOMEM;
 997
 998         if (unlikely(vq->vq.num_free < 1)) {
 999                 pr_debug("Can't add buf len 1 - avail = 0\n");
1000                 kfree(desc);
1001                 END_USE(vq);
1002                 return -ENOSPC;
1003         }
1004
1005         i = 0;
1006         id = vq->free_head;
1007         BUG_ON(id == vq->packed.vring.num);
1008
1009         for (n = 0; n < out_sgs + in_sgs; n++) {
1010                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1011                         addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1012                                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
1013                         if (vring_mapping_error(vq, addr))
1014                                 goto unmap_release;
1015
1016                         desc[i].flags = cpu_to_le16(n < out_sgs ?
1017                                                 0 : VRING_DESC_F_WRITE);
1018                         desc[i].addr = cpu_to_le64(addr);
1019                         desc[i].len = cpu_to_le32(sg->length);
1020                         i++;
1021                 }
1022         }
1023
1024         /* Now that the indirect table is filled in, map it. */
1025         addr = vring_map_single(vq, desc,
1026                         total_sg * sizeof(struct vring_packed_desc),
1027                         DMA_TO_DEVICE);
1028         if (vring_mapping_error(vq, addr))
1029                 goto unmap_release;
1030
1031         vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1032         vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1033                                 sizeof(struct vring_packed_desc));
1034         vq->packed.vring.desc[head].id = cpu_to_le16(id);
1035
1036         if (vq->use_dma_api) {
1037                 vq->packed.desc_extra[id].addr = addr;
1038                 vq->packed.desc_extra[id].len = total_sg *
1039                                 sizeof(struct vring_packed_desc);
1040                 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1041                                                   vq->packed.avail_used_flags;
1042         }
1043
1044         /*
1045          * A driver MUST NOT make the first descriptor in the list
1046          * available before all subsequent descriptors comprising
1047          * the list are made available.
1048          */
1049         virtio_wmb(vq->weak_barriers);
1050         vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1051                                                 vq->packed.avail_used_flags);
1052
1053         /* We're using some buffers from the free list. */
1054         vq->vq.num_free -= 1;
1055
1056         /* Update free pointer */
1057         n = head + 1;
1058         if (n >= vq->packed.vring.num) {
1059                 n = 0;
1060                 vq->packed.avail_wrap_counter ^= 1;
1061                 vq->packed.avail_used_flags ^=
1062                                 1 << VRING_PACKED_DESC_F_AVAIL |
1063                                 1 << VRING_PACKED_DESC_F_USED;
1064         }
1065         vq->packed.next_avail_idx = n;
1066         vq->free_head = vq->packed.desc_state[id].next;
1067
1068         /* Store token and indirect buffer state. */
1069         vq->packed.desc_state[id].num = 1;
1070         vq->packed.desc_state[id].data = data;
1071         vq->packed.desc_state[id].indir_desc = desc;
1072         vq->packed.desc_state[id].last = id;
1073
1074         vq->num_added += 1;
1075
1076         pr_debug("Added buffer head %i to %p\n", head, vq);
1077         END_USE(vq);
1078
1079         return 0;
1080
1081 unmap_release:
1082         err_idx = i;
1083
1084         for (i = 0; i < err_idx; i++)
1085                 vring_unmap_desc_packed(vq, &desc[i]);
1086
1087         kfree(desc);
1088
1089         END_USE(vq);
1090         return -ENOMEM;
1091 }
1092
1093 static inline int virtqueue_add_packed(struct virtqueue *_vq,
1094                                        struct scatterlist *sgs[],
1095                                        unsigned int total_sg,
1096                                        unsigned int out_sgs,
1097                                        unsigned int in_sgs,
1098                                        void *data,
1099                                        void *ctx,
1100                                        gfp_t gfp)
1101 {
1102         struct vring_virtqueue *vq = to_vvq(_vq);
1103         struct vring_packed_desc *desc;
1104         struct scatterlist *sg;
1105         unsigned int i, n, c, descs_used, err_idx;
1106         __le16 head_flags, flags;
1107         u16 head, id, prev, curr, avail_used_flags;
1108         int err;
1109
1110         START_USE(vq);
1111
1112         BUG_ON(data == NULL);
1113         BUG_ON(ctx && vq->indirect);
1114
1115         if (unlikely(vq->broken)) {
1116                 END_USE(vq);
1117                 return -EIO;
1118         }
1119
1120         LAST_ADD_TIME_UPDATE(vq);
1121
1122         BUG_ON(total_sg == 0);
1123
1124         if (virtqueue_use_indirect(_vq, total_sg)) {
1125                 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1126                                                     in_sgs, data, gfp);
1127                 if (err != -ENOMEM) {
1128                         END_USE(vq);
1129                         return err;
1130                 }
1131
1132                 /* fall back on direct */
1133         }
1134
1135         head = vq->packed.next_avail_idx;
1136         avail_used_flags = vq->packed.avail_used_flags;
1137
1138         WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1139
1140         desc = vq->packed.vring.desc;
1141         i = head;
1142         descs_used = total_sg;
1143
1144         if (unlikely(vq->vq.num_free < descs_used)) {
1145                 pr_debug("Can't add buf len %i - avail = %i\n",
1146                          descs_used, vq->vq.num_free);
1147                 END_USE(vq);
1148                 return -ENOSPC;
1149         }
1150
1151         id = vq->free_head;
1152         BUG_ON(id == vq->packed.vring.num);
1153
1154         curr = id;
1155         c = 0;
1156         for (n = 0; n < out_sgs + in_sgs; n++) {
1157                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1158                         dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1159                                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
1160                         if (vring_mapping_error(vq, addr))
1161                                 goto unmap_release;
1162
1163                         flags = cpu_to_le16(vq->packed.avail_used_flags |
1164                                     (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1165                                     (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1166                         if (i == head)
1167                                 head_flags = flags;
1168                         else
1169                                 desc[i].flags = flags;
1170
1171                         desc[i].addr = cpu_to_le64(addr);
1172                         desc[i].len = cpu_to_le32(sg->length);
1173                         desc[i].id = cpu_to_le16(id);
1174
1175                         if (unlikely(vq->use_dma_api)) {
1176                                 vq->packed.desc_extra[curr].addr = addr;
1177                                 vq->packed.desc_extra[curr].len = sg->length;
1178                                 vq->packed.desc_extra[curr].flags =
1179                                         le16_to_cpu(flags);
1180                         }
1181                         prev = curr;
1182                         curr = vq->packed.desc_state[curr].next;
1183
1184                         if ((unlikely(++i >= vq->packed.vring.num))) {
1185                                 i = 0;
1186                                 vq->packed.avail_used_flags ^=
1187                                         1 << VRING_PACKED_DESC_F_AVAIL |
1188                                         1 << VRING_PACKED_DESC_F_USED;
1189                         }
1190                 }
1191         }
1192
1193         if (i <= head)
1194                 vq->packed.avail_wrap_counter ^= 1;
1195
1196         /* We're using some buffers from the free list. */
1197         vq->vq.num_free -= descs_used;
1198
1199         /* Update free pointer */
1200         vq->packed.next_avail_idx = i;
1201         vq->free_head = curr;
1202
1203         /* Store token. */
1204         vq->packed.desc_state[id].num = descs_used;
1205         vq->packed.desc_state[id].data = data;
1206         vq->packed.desc_state[id].indir_desc = ctx;
1207         vq->packed.desc_state[id].last = prev;
1208
1209         /*
1210          * A driver MUST NOT make the first descriptor in the list
1211          * available before all subsequent descriptors comprising
1212          * the list are made available.
1213          */
1214         virtio_wmb(vq->weak_barriers);
1215         vq->packed.vring.desc[head].flags = head_flags;
1216         vq->num_added += descs_used;
1217
1218         pr_debug("Added buffer head %i to %p\n", head, vq);
1219         END_USE(vq);
1220
1221         return 0;
1222
1223 unmap_release:
1224         err_idx = i;
1225         i = head;
1226
1227         vq->packed.avail_used_flags = avail_used_flags;
1228
1229         for (n = 0; n < total_sg; n++) {
1230                 if (i == err_idx)
1231                         break;
1232                 vring_unmap_desc_packed(vq, &desc[i]);
1233                 i++;
1234                 if (i >= vq->packed.vring.num)
1235                         i = 0;
1236         }
1237
1238         END_USE(vq);
1239         return -EIO;
1240 }
1241
1242 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1243 {
1244         struct vring_virtqueue *vq = to_vvq(_vq);
1245         u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1246         bool needs_kick;
1247         union {
1248                 struct {
1249                         __le16 off_wrap;
1250                         __le16 flags;
1251                 };
1252                 u32 u32;
1253         } snapshot;
1254
1255         START_USE(vq);
1256
1257         /*
1258          * We need to expose the new flags value before checking notification
1259          * suppressions.
1260          */
1261         virtio_mb(vq->weak_barriers);
1262
1263         old = vq->packed.next_avail_idx - vq->num_added;
1264         new = vq->packed.next_avail_idx;
1265         vq->num_added = 0;
1266
1267         snapshot.u32 = *(u32 *)vq->packed.vring.device;
1268         flags = le16_to_cpu(snapshot.flags);
1269
1270         LAST_ADD_TIME_CHECK(vq);
1271         LAST_ADD_TIME_INVALID(vq);
1272
1273         if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1274                 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1275                 goto out;
1276         }
1277
1278         off_wrap = le16_to_cpu(snapshot.off_wrap);
1279
1280         wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1281         event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1282         if (wrap_counter != vq->packed.avail_wrap_counter)
1283                 event_idx -= vq->packed.vring.num;
1284
1285         needs_kick = vring_need_event(event_idx, new, old);
1286 out:
1287         END_USE(vq);
1288         return needs_kick;
1289 }
1290
1291 static void detach_buf_packed(struct vring_virtqueue *vq,
1292                               unsigned int id, void **ctx)
1293 {
1294         struct vring_desc_state_packed *state = NULL;
1295         struct vring_packed_desc *desc;
1296         unsigned int i, curr;
1297
1298         state = &vq->packed.desc_state[id];
1299
1300         /* Clear data ptr. */
1301         state->data = NULL;
1302
1303         vq->packed.desc_state[state->last].next = vq->free_head;
1304         vq->free_head = id;
1305         vq->vq.num_free += state->num;
1306
1307         if (unlikely(vq->use_dma_api)) {
1308                 curr = id;
1309                 for (i = 0; i < state->num; i++) {
1310                         vring_unmap_state_packed(vq,
1311                                 &vq->packed.desc_extra[curr]);
1312                         curr = vq->packed.desc_state[curr].next;
1313                 }
1314         }
1315
1316         if (vq->indirect) {
1317                 u32 len;
1318
1319                 /* Free the indirect table, if any, now that it's unmapped. */
1320                 desc = state->indir_desc;
1321                 if (!desc)
1322                         return;
1323
1324                 if (vq->use_dma_api) {
1325                         len = vq->packed.desc_extra[id].len;
1326                         for (i = 0; i < len / sizeof(struct vring_packed_desc);
1327                                         i++)
1328                                 vring_unmap_desc_packed(vq, &desc[i]);
1329                 }
1330                 kfree(desc);
1331                 state->indir_desc = NULL;
1332         } else if (ctx) {
1333                 *ctx = state->indir_desc;
1334         }
1335 }
1336
1337 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1338                                        u16 idx, bool used_wrap_counter)
1339 {
1340         bool avail, used;
1341         u16 flags;
1342
1343         flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1344         avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1345         used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1346
1347         return avail == used && used == used_wrap_counter;
1348 }
1349
1350 static inline bool more_used_packed(const struct vring_virtqueue *vq)
1351 {
1352         return is_used_desc_packed(vq, vq->last_used_idx,
1353                         vq->packed.used_wrap_counter);
1354 }
1355
1356 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1357                                           unsigned int *len,
1358                                           void **ctx)
1359 {
1360         struct vring_virtqueue *vq = to_vvq(_vq);
1361         u16 last_used, id;
1362         void *ret;
1363
1364         START_USE(vq);
1365
1366         if (unlikely(vq->broken)) {
1367                 END_USE(vq);
1368                 return NULL;
1369         }
1370
1371         if (!more_used_packed(vq)) {
1372                 pr_debug("No more buffers in queue\n");
1373                 END_USE(vq);
1374                 return NULL;
1375         }
1376
1377         /* Only get used elements after they have been exposed by host. */
1378         virtio_rmb(vq->weak_barriers);
1379
1380         last_used = vq->last_used_idx;
1381         id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1382         *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1383
1384         if (unlikely(id >= vq->packed.vring.num)) {
1385                 BAD_RING(vq, "id %u out of range\n", id);
1386                 return NULL;
1387         }
1388         if (unlikely(!vq->packed.desc_state[id].data)) {
1389                 BAD_RING(vq, "id %u is not a head!\n", id);
1390                 return NULL;
1391         }
1392
1393         /* detach_buf_packed clears data, so grab it now. */
1394         ret = vq->packed.desc_state[id].data;
1395         detach_buf_packed(vq, id, ctx);
1396
1397         vq->last_used_idx += vq->packed.desc_state[id].num;
1398         if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1399                 vq->last_used_idx -= vq->packed.vring.num;
1400                 vq->packed.used_wrap_counter ^= 1;
1401         }
1402
1403         /*
1404          * If we expect an interrupt for the next entry, tell host
1405          * by writing event index and flush out the write before
1406          * the read in the next get_buf call.
1407          */
1408         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1409                 virtio_store_mb(vq->weak_barriers,
1410                                 &vq->packed.vring.driver->off_wrap,
1411                                 cpu_to_le16(vq->last_used_idx |
1412                                         (vq->packed.used_wrap_counter <<
1413                                          VRING_PACKED_EVENT_F_WRAP_CTR)));
1414
1415         LAST_ADD_TIME_INVALID(vq);
1416
1417         END_USE(vq);
1418         return ret;
1419 }
1420
1421 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1422 {
1423         struct vring_virtqueue *vq = to_vvq(_vq);
1424
1425         if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1426                 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1427                 vq->packed.vring.driver->flags =
1428                         cpu_to_le16(vq->packed.event_flags_shadow);
1429         }
1430 }
1431
1432 static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1433 {
1434         struct vring_virtqueue *vq = to_vvq(_vq);
1435
1436         START_USE(vq);
1437
1438         /*
1439          * We optimistically turn back on interrupts, then check if there was
1440          * more to do.
1441          */
1442
1443         if (vq->event) {
1444                 vq->packed.vring.driver->off_wrap =
1445                         cpu_to_le16(vq->last_used_idx |
1446                                 (vq->packed.used_wrap_counter <<
1447                                  VRING_PACKED_EVENT_F_WRAP_CTR));
1448                 /*
1449                  * We need to update event offset and event wrap
1450                  * counter first before updating event flags.
1451                  */
1452                 virtio_wmb(vq->weak_barriers);
1453         }
1454
1455         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1456                 vq->packed.event_flags_shadow = vq->event ?
1457                                 VRING_PACKED_EVENT_FLAG_DESC :
1458                                 VRING_PACKED_EVENT_FLAG_ENABLE;
1459                 vq->packed.vring.driver->flags =
1460                                 cpu_to_le16(vq->packed.event_flags_shadow);
1461         }
1462
1463         END_USE(vq);
1464         return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1465                         VRING_PACKED_EVENT_F_WRAP_CTR);
1466 }
1467
1468 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1469 {
1470         struct vring_virtqueue *vq = to_vvq(_vq);
1471         bool wrap_counter;
1472         u16 used_idx;
1473
1474         wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1475         used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1476
1477         return is_used_desc_packed(vq, used_idx, wrap_counter);
1478 }
1479
1480 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1481 {
1482         struct vring_virtqueue *vq = to_vvq(_vq);
1483         u16 used_idx, wrap_counter;
1484         u16 bufs;
1485
1486         START_USE(vq);
1487
1488         /*
1489          * We optimistically turn back on interrupts, then check if there was
1490          * more to do.
1491          */
1492
1493         if (vq->event) {
1494                 /* TODO: tune this threshold */
1495                 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1496                 wrap_counter = vq->packed.used_wrap_counter;
1497
1498                 used_idx = vq->last_used_idx + bufs;
1499                 if (used_idx >= vq->packed.vring.num) {
1500                         used_idx -= vq->packed.vring.num;
1501                         wrap_counter ^= 1;
1502                 }
1503
1504                 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1505                         (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1506
1507                 /*
1508                  * We need to update event offset and event wrap
1509                  * counter first before updating event flags.
1510                  */
1511                 virtio_wmb(vq->weak_barriers);
1512         }
1513
1514         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1515                 vq->packed.event_flags_shadow = vq->event ?
1516                                 VRING_PACKED_EVENT_FLAG_DESC :
1517                                 VRING_PACKED_EVENT_FLAG_ENABLE;
1518                 vq->packed.vring.driver->flags =
1519                                 cpu_to_le16(vq->packed.event_flags_shadow);
1520         }
1521
1522         /*
1523          * We need to update event suppression structure first
1524          * before re-checking for more used buffers.
1525          */
1526         virtio_mb(vq->weak_barriers);
1527
1528         if (is_used_desc_packed(vq,
1529                                 vq->last_used_idx,
1530                                 vq->packed.used_wrap_counter)) {
1531                 END_USE(vq);
1532                 return false;
1533         }
1534
1535         END_USE(vq);
1536         return true;
1537 }
1538
1539 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1540 {
1541         struct vring_virtqueue *vq = to_vvq(_vq);
1542         unsigned int i;
1543         void *buf;
1544
1545         START_USE(vq);
1546
1547         for (i = 0; i < vq->packed.vring.num; i++) {
1548                 if (!vq->packed.desc_state[i].data)
1549                         continue;
1550                 /* detach_buf clears data, so grab it now. */
1551                 buf = vq->packed.desc_state[i].data;
1552                 detach_buf_packed(vq, i, NULL);
1553                 END_USE(vq);
1554                 return buf;
1555         }
1556         /* That should have freed everything. */
1557         BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1558
1559         END_USE(vq);
1560         return NULL;
1561 }
1562
1563 static struct virtqueue *vring_create_virtqueue_packed(
1564         unsigned int index,
1565         unsigned int num,
1566         unsigned int vring_align,
1567         struct virtio_device *vdev,
1568         bool weak_barriers,
1569         bool may_reduce_num,
1570         bool context,
1571         bool (*notify)(struct virtqueue *),
1572         void (*callback)(struct virtqueue *),
1573         const char *name)
1574 {
1575         struct vring_virtqueue *vq;
1576         struct vring_packed_desc *ring;
1577         struct vring_packed_desc_event *driver, *device;
1578         dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1579         size_t ring_size_in_bytes, event_size_in_bytes;
1580         unsigned int i;
1581
1582         ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1583
1584         ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1585                                  &ring_dma_addr,
1586                                  GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1587         if (!ring)
1588                 goto err_ring;
1589
1590         event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1591
1592         driver = vring_alloc_queue(vdev, event_size_in_bytes,
1593                                    &driver_event_dma_addr,
1594                                    GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1595         if (!driver)
1596                 goto err_driver;
1597
1598         device = vring_alloc_queue(vdev, event_size_in_bytes,
1599                                    &device_event_dma_addr,
1600                                    GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1601         if (!device)
1602                 goto err_device;
1603
1604         vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1605         if (!vq)
1606                 goto err_vq;
1607
1608         vq->vq.callback = callback;
1609         vq->vq.vdev = vdev;
1610         vq->vq.name = name;
1611         vq->vq.num_free = num;
1612         vq->vq.index = index;
1613         vq->we_own_ring = true;
1614         vq->notify = notify;
1615         vq->weak_barriers = weak_barriers;
1616         vq->broken = false;
1617         vq->last_used_idx = 0;
1618         vq->num_added = 0;
1619         vq->packed_ring = true;
1620         vq->use_dma_api = vring_use_dma_api(vdev);
1621 #ifdef DEBUG
1622         vq->in_use = false;
1623         vq->last_add_time_valid = false;
1624 #endif
1625
1626         vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1627                 !context;
1628         vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1629
1630         if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1631                 vq->weak_barriers = false;
1632
1633         vq->packed.ring_dma_addr = ring_dma_addr;
1634         vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1635         vq->packed.device_event_dma_addr = device_event_dma_addr;
1636
1637         vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1638         vq->packed.event_size_in_bytes = event_size_in_bytes;
1639
1640         vq->packed.vring.num = num;
1641         vq->packed.vring.desc = ring;
1642         vq->packed.vring.driver = driver;
1643         vq->packed.vring.device = device;
1644
1645         vq->packed.next_avail_idx = 0;
1646         vq->packed.avail_wrap_counter = 1;
1647         vq->packed.used_wrap_counter = 1;
1648         vq->packed.event_flags_shadow = 0;
1649         vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1650
1651         vq->packed.desc_state = kmalloc_array(num,
1652                         sizeof(struct vring_desc_state_packed),
1653                         GFP_KERNEL);
1654         if (!vq->packed.desc_state)
1655                 goto err_desc_state;
1656
1657         memset(vq->packed.desc_state, 0,
1658                 num * sizeof(struct vring_desc_state_packed));
1659
1660         /* Put everything in free lists. */
1661         vq->free_head = 0;
1662         for (i = 0; i < num-1; i++)
1663                 vq->packed.desc_state[i].next = i + 1;
1664
1665         vq->packed.desc_extra = kmalloc_array(num,
1666                         sizeof(struct vring_desc_extra_packed),
1667                         GFP_KERNEL);
1668         if (!vq->packed.desc_extra)
1669                 goto err_desc_extra;
1670
1671         memset(vq->packed.desc_extra, 0,
1672                 num * sizeof(struct vring_desc_extra_packed));
1673
1674         /* No callback?  Tell other side not to bother us. */
1675         if (!callback) {
1676                 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1677                 vq->packed.vring.driver->flags =
1678                         cpu_to_le16(vq->packed.event_flags_shadow);
1679         }
1680
1681         spin_lock(&vdev->vqs_list_lock);
1682         list_add_tail(&vq->vq.list, &vdev->vqs);
1683         spin_unlock(&vdev->vqs_list_lock);
1684         return &vq->vq;
1685
1686 err_desc_extra:
1687         kfree(vq->packed.desc_state);
1688 err_desc_state:
1689         kfree(vq);
1690 err_vq:
1691         vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr);
1692 err_device:
1693         vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr);
1694 err_driver:
1695         vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1696 err_ring:
1697         return NULL;
1698 }
1699
1700
1701 /*
1702  * Generic functions and exported symbols.
1703  */
1704
1705 static inline int virtqueue_add(struct virtqueue *_vq,
1706                                 struct scatterlist *sgs[],
1707                                 unsigned int total_sg,
1708                                 unsigned int out_sgs,
1709                                 unsigned int in_sgs,
1710                                 void *data,
1711                                 void *ctx,
1712                                 gfp_t gfp)
1713 {
1714         struct vring_virtqueue *vq = to_vvq(_vq);
1715
1716         return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1717                                         out_sgs, in_sgs, data, ctx, gfp) :
1718                                  virtqueue_add_split(_vq, sgs, total_sg,
1719                                         out_sgs, in_sgs, data, ctx, gfp);
1720 }
1721
1722 /**
1723  * virtqueue_add_sgs - expose buffers to other end
1724  * @_vq: the struct virtqueue we're talking about.
1725  * @sgs: array of terminated scatterlists.
1726  * @out_sgs: the number of scatterlists readable by other side
1727  * @in_sgs: the number of scatterlists which are writable (after readable ones)
1728  * @data: the token identifying the buffer.
1729  * @gfp: how to do memory allocations (if necessary).
1730  *
1731  * Caller must ensure we don't call this with other virtqueue operations
1732  * at the same time (except where noted).
1733  *
1734  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1735  */
1736 int virtqueue_add_sgs(struct virtqueue *_vq,
1737                       struct scatterlist *sgs[],
1738                       unsigned int out_sgs,
1739                       unsigned int in_sgs,
1740                       void *data,
1741                       gfp_t gfp)
1742 {
1743         unsigned int i, total_sg = 0;
1744
1745         /* Count them first. */
1746         for (i = 0; i < out_sgs + in_sgs; i++) {
1747                 struct scatterlist *sg;
1748
1749                 for (sg = sgs[i]; sg; sg = sg_next(sg))
1750                         total_sg++;
1751         }
1752         return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1753                              data, NULL, gfp);
1754 }
1755 EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1756
1757 /**
1758  * virtqueue_add_outbuf - expose output buffers to other end
1759  * @vq: the struct virtqueue we're talking about.
1760  * @sg: scatterlist (must be well-formed and terminated!)
1761  * @num: the number of entries in @sg readable by other side
1762  * @data: the token identifying the buffer.
1763  * @gfp: how to do memory allocations (if necessary).
1764  *
1765  * Caller must ensure we don't call this with other virtqueue operations
1766  * at the same time (except where noted).
1767  *
1768  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1769  */
1770 int virtqueue_add_outbuf(struct virtqueue *vq,
1771                          struct scatterlist *sg, unsigned int num,
1772                          void *data,
1773                          gfp_t gfp)
1774 {
1775         return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1776 }
1777 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1778
1779 /**
1780  * virtqueue_add_inbuf - expose input buffers to other end
1781  * @vq: the struct virtqueue we're talking about.
1782  * @sg: scatterlist (must be well-formed and terminated!)
1783  * @num: the number of entries in @sg writable by other side
1784  * @data: the token identifying the buffer.
1785  * @gfp: how to do memory allocations (if necessary).
1786  *
1787  * Caller must ensure we don't call this with other virtqueue operations
1788  * at the same time (except where noted).
1789  *
1790  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1791  */
1792 int virtqueue_add_inbuf(struct virtqueue *vq,
1793                         struct scatterlist *sg, unsigned int num,
1794                         void *data,
1795                         gfp_t gfp)
1796 {
1797         return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1798 }
1799 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1800
1801 /**
1802  * virtqueue_add_inbuf_ctx - expose input buffers to other end
1803  * @vq: the struct virtqueue we're talking about.
1804  * @sg: scatterlist (must be well-formed and terminated!)
1805  * @num: the number of entries in @sg writable by other side
1806  * @data: the token identifying the buffer.
1807  * @ctx: extra context for the token
1808  * @gfp: how to do memory allocations (if necessary).
1809  *
1810  * Caller must ensure we don't call this with other virtqueue operations
1811  * at the same time (except where noted).
1812  *
1813  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1814  */
1815 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1816                         struct scatterlist *sg, unsigned int num,
1817                         void *data,
1818                         void *ctx,
1819                         gfp_t gfp)
1820 {
1821         return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1822 }
1823 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1824
1825 /**
1826  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1827  * @_vq: the struct virtqueue
1828  *
1829  * Instead of virtqueue_kick(), you can do:
1830  *      if (virtqueue_kick_prepare(vq))
1831  *              virtqueue_notify(vq);
1832  *
1833  * This is sometimes useful because the virtqueue_kick_prepare() needs
1834  * to be serialized, but the actual virtqueue_notify() call does not.
1835  */
1836 bool virtqueue_kick_prepare(struct virtqueue *_vq)
1837 {
1838         struct vring_virtqueue *vq = to_vvq(_vq);
1839
1840         return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1841                                  virtqueue_kick_prepare_split(_vq);
1842 }
1843 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1844
1845 /**
1846  * virtqueue_notify - second half of split virtqueue_kick call.
1847  * @_vq: the struct virtqueue
1848  *
1849  * This does not need to be serialized.
1850  *
1851  * Returns false if host notify failed or queue is broken, otherwise true.
1852  */
1853 bool virtqueue_notify(struct virtqueue *_vq)
1854 {
1855         struct vring_virtqueue *vq = to_vvq(_vq);
1856
1857         if (unlikely(vq->broken))
1858                 return false;
1859
1860         /* Prod other side to tell it about changes. */
1861         if (!vq->notify(_vq)) {
1862                 vq->broken = true;
1863                 return false;
1864         }
1865         return true;
1866 }
1867 EXPORT_SYMBOL_GPL(virtqueue_notify);
1868
1869 /**
1870  * virtqueue_kick - update after add_buf
1871  * @vq: the struct virtqueue
1872  *
1873  * After one or more virtqueue_add_* calls, invoke this to kick
1874  * the other side.
1875  *
1876  * Caller must ensure we don't call this with other virtqueue
1877  * operations at the same time (except where noted).
1878  *
1879  * Returns false if kick failed, otherwise true.
1880  */
1881 bool virtqueue_kick(struct virtqueue *vq)
1882 {
1883         if (virtqueue_kick_prepare(vq))
1884                 return virtqueue_notify(vq);
1885         return true;
1886 }
1887 EXPORT_SYMBOL_GPL(virtqueue_kick);
1888
1889 /**
1890  * virtqueue_get_buf - get the next used buffer
1891  * @_vq: the struct virtqueue we're talking about.
1892  * @len: the length written into the buffer
1893  * @ctx: extra context for the token
1894  *
1895  * If the device wrote data into the buffer, @len will be set to the
1896  * amount written.  This means you don't need to clear the buffer
1897  * beforehand to ensure there's no data leakage in the case of short
1898  * writes.
1899  *
1900  * Caller must ensure we don't call this with other virtqueue
1901  * operations at the same time (except where noted).
1902  *
1903  * Returns NULL if there are no used buffers, or the "data" token
1904  * handed to virtqueue_add_*().
1905  */
1906 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1907                             void **ctx)
1908 {
1909         struct vring_virtqueue *vq = to_vvq(_vq);
1910
1911         return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
1912                                  virtqueue_get_buf_ctx_split(_vq, len, ctx);
1913 }
1914 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
1915
1916 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
1917 {
1918         return virtqueue_get_buf_ctx(_vq, len, NULL);
1919 }
1920 EXPORT_SYMBOL_GPL(virtqueue_get_buf);
1921 /**
1922  * virtqueue_disable_cb - disable callbacks
1923  * @_vq: the struct virtqueue we're talking about.
1924  *
1925  * Note that this is not necessarily synchronous, hence unreliable and only
1926  * useful as an optimization.
1927  *
1928  * Unlike other operations, this need not be serialized.
1929  */
1930 void virtqueue_disable_cb(struct virtqueue *_vq)
1931 {
1932         struct vring_virtqueue *vq = to_vvq(_vq);
1933
1934         if (vq->packed_ring)
1935                 virtqueue_disable_cb_packed(_vq);
1936         else
1937                 virtqueue_disable_cb_split(_vq);
1938 }
1939 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1940
1941 /**
1942  * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
1943  * @_vq: the struct virtqueue we're talking about.
1944  *
1945  * This re-enables callbacks; it returns current queue state
1946  * in an opaque unsigned value. This value should be later tested by
1947  * virtqueue_poll, to detect a possible race between the driver checking for
1948  * more work, and enabling callbacks.
1949  *
1950  * Caller must ensure we don't call this with other virtqueue
1951  * operations at the same time (except where noted).
1952  */
1953 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1954 {
1955         struct vring_virtqueue *vq = to_vvq(_vq);
1956
1957         return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1958                                  virtqueue_enable_cb_prepare_split(_vq);
1959 }
1960 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1961
1962 /**
1963  * virtqueue_poll - query pending used buffers
1964  * @_vq: the struct virtqueue we're talking about.
1965  * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
1966  *
1967  * Returns "true" if there are pending used buffers in the queue.
1968  *
1969  * This does not need to be serialized.
1970  */
1971 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
1972 {
1973         struct vring_virtqueue *vq = to_vvq(_vq);
1974
1975         if (unlikely(vq->broken))
1976                 return false;
1977
1978         virtio_mb(vq->weak_barriers);
1979         return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
1980                                  virtqueue_poll_split(_vq, last_used_idx);
1981 }
1982 EXPORT_SYMBOL_GPL(virtqueue_poll);
1983
1984 /**
1985  * virtqueue_enable_cb - restart callbacks after disable_cb.
1986  * @_vq: the struct virtqueue we're talking about.
1987  *
1988  * This re-enables callbacks; it returns "false" if there are pending
1989  * buffers in the queue, to detect a possible race between the driver
1990  * checking for more work, and enabling callbacks.
1991  *
1992  * Caller must ensure we don't call this with other virtqueue
1993  * operations at the same time (except where noted).
1994  */
1995 bool virtqueue_enable_cb(struct virtqueue *_vq)
1996 {
1997         unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
1998
1999         return !virtqueue_poll(_vq, last_used_idx);
2000 }
2001 EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2002
2003 /**
2004  * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2005  * @_vq: the struct virtqueue we're talking about.
2006  *
2007  * This re-enables callbacks but hints to the other side to delay
2008  * interrupts until most of the available buffers have been processed;
2009  * it returns "false" if there are many pending buffers in the queue,
2010  * to detect a possible race between the driver checking for more work,
2011  * and enabling callbacks.
2012  *
2013  * Caller must ensure we don't call this with other virtqueue
2014  * operations at the same time (except where noted).
2015  */
2016 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2017 {
2018         struct vring_virtqueue *vq = to_vvq(_vq);
2019
2020         return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2021                                  virtqueue_enable_cb_delayed_split(_vq);
2022 }
2023 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2024
2025 /**
2026  * virtqueue_detach_unused_buf - detach first unused buffer
2027  * @_vq: the struct virtqueue we're talking about.
2028  *
2029  * Returns NULL or the "data" token handed to virtqueue_add_*().
2030  * This is not valid on an active queue; it is useful only for device
2031  * shutdown.
2032  */
2033 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2034 {
2035         struct vring_virtqueue *vq = to_vvq(_vq);
2036
2037         return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2038                                  virtqueue_detach_unused_buf_split(_vq);
2039 }
2040 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2041
2042 static inline bool more_used(const struct vring_virtqueue *vq)
2043 {
2044         return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2045 }
2046
2047 irqreturn_t vring_interrupt(int irq, void *_vq)
2048 {
2049         struct vring_virtqueue *vq = to_vvq(_vq);
2050
2051         if (!more_used(vq)) {
2052                 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2053                 return IRQ_NONE;
2054         }
2055
2056         if (unlikely(vq->broken))
2057                 return IRQ_HANDLED;
2058
2059         pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2060         if (vq->vq.callback)
2061                 vq->vq.callback(&vq->vq);
2062
2063         return IRQ_HANDLED;
2064 }
2065 EXPORT_SYMBOL_GPL(vring_interrupt);
2066
2067 /* Only available for split ring */
2068 struct virtqueue *__vring_new_virtqueue(unsigned int index,
2069                                         struct vring vring,
2070                                         struct virtio_device *vdev,
2071                                         bool weak_barriers,
2072                                         bool context,
2073                                         bool (*notify)(struct virtqueue *),
2074                                         void (*callback)(struct virtqueue *),
2075                                         const char *name)
2076 {
2077         unsigned int i;
2078         struct vring_virtqueue *vq;
2079
2080         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2081                 return NULL;
2082
2083         vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2084         if (!vq)
2085                 return NULL;
2086
2087         vq->packed_ring = false;
2088         vq->vq.callback = callback;
2089         vq->vq.vdev = vdev;
2090         vq->vq.name = name;
2091         vq->vq.num_free = vring.num;
2092         vq->vq.index = index;
2093         vq->we_own_ring = false;
2094         vq->notify = notify;
2095         vq->weak_barriers = weak_barriers;
2096         vq->broken = false;
2097         vq->last_used_idx = 0;
2098         vq->num_added = 0;
2099         vq->use_dma_api = vring_use_dma_api(vdev);
2100 #ifdef DEBUG
2101         vq->in_use = false;
2102         vq->last_add_time_valid = false;
2103 #endif
2104
2105         vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2106                 !context;
2107         vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2108
2109         if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2110                 vq->weak_barriers = false;
2111
2112         vq->split.queue_dma_addr = 0;
2113         vq->split.queue_size_in_bytes = 0;
2114
2115         vq->split.vring = vring;
2116         vq->split.avail_flags_shadow = 0;
2117         vq->split.avail_idx_shadow = 0;
2118
2119         /* No callback?  Tell other side not to bother us. */
2120         if (!callback) {
2121                 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2122                 if (!vq->event)
2123                         vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2124                                         vq->split.avail_flags_shadow);
2125         }
2126
2127         vq->split.desc_state = kmalloc_array(vring.num,
2128                         sizeof(struct vring_desc_state_split), GFP_KERNEL);
2129         if (!vq->split.desc_state) {
2130                 kfree(vq);
2131                 return NULL;
2132         }
2133
2134         /* Put everything in free lists. */
2135         vq->free_head = 0;
2136         for (i = 0; i < vring.num-1; i++)
2137                 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
2138         memset(vq->split.desc_state, 0, vring.num *
2139                         sizeof(struct vring_desc_state_split));
2140
2141         spin_lock(&vdev->vqs_list_lock);
2142         list_add_tail(&vq->vq.list, &vdev->vqs);
2143         spin_unlock(&vdev->vqs_list_lock);
2144         return &vq->vq;
2145 }
2146 EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2147
2148 struct virtqueue *vring_create_virtqueue(
2149         unsigned int index,
2150         unsigned int num,
2151         unsigned int vring_align,
2152         struct virtio_device *vdev,
2153         bool weak_barriers,
2154         bool may_reduce_num,
2155         bool context,
2156         bool (*notify)(struct virtqueue *),
2157         void (*callback)(struct virtqueue *),
2158         const char *name)
2159 {
2160
2161         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2162                 return vring_create_virtqueue_packed(index, num, vring_align,
2163                                 vdev, weak_barriers, may_reduce_num,
2164                                 context, notify, callback, name);
2165
2166         return vring_create_virtqueue_split(index, num, vring_align,
2167                         vdev, weak_barriers, may_reduce_num,
2168                         context, notify, callback, name);
2169 }
2170 EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2171
2172 /* Only available for split ring */
2173 struct virtqueue *vring_new_virtqueue(unsigned int index,
2174                                       unsigned int num,
2175                                       unsigned int vring_align,
2176                                       struct virtio_device *vdev,
2177                                       bool weak_barriers,
2178                                       bool context,
2179                                       void *pages,
2180                                       bool (*notify)(struct virtqueue *vq),
2181                                       void (*callback)(struct virtqueue *vq),
2182                                       const char *name)
2183 {
2184         struct vring vring;
2185
2186         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2187                 return NULL;
2188
2189         vring_init(&vring, num, pages, vring_align);
2190         return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2191                                      notify, callback, name);
2192 }
2193 EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2194
2195 void vring_del_virtqueue(struct virtqueue *_vq)
2196 {
2197         struct vring_virtqueue *vq = to_vvq(_vq);
2198
2199         if (vq->we_own_ring) {
2200                 if (vq->packed_ring) {
2201                         vring_free_queue(vq->vq.vdev,
2202                                          vq->packed.ring_size_in_bytes,
2203                                          vq->packed.vring.desc,
2204                                          vq->packed.ring_dma_addr);
2205
2206                         vring_free_queue(vq->vq.vdev,
2207                                          vq->packed.event_size_in_bytes,
2208                                          vq->packed.vring.driver,
2209                                          vq->packed.driver_event_dma_addr);
2210
2211                         vring_free_queue(vq->vq.vdev,
2212                                          vq->packed.event_size_in_bytes,
2213                                          vq->packed.vring.device,
2214                                          vq->packed.device_event_dma_addr);
2215
2216                         kfree(vq->packed.desc_state);
2217                         kfree(vq->packed.desc_extra);
2218                 } else {
2219                         vring_free_queue(vq->vq.vdev,
2220                                          vq->split.queue_size_in_bytes,
2221                                          vq->split.vring.desc,
2222                                          vq->split.queue_dma_addr);
2223                 }
2224         }
2225         if (!vq->packed_ring)
2226                 kfree(vq->split.desc_state);
2227         spin_lock(&vq->vq.vdev->vqs_list_lock);
2228         list_del(&_vq->list);
2229         spin_unlock(&vq->vq.vdev->vqs_list_lock);
2230         kfree(vq);
2231 }
2232 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2233
2234 /* Manipulates transport-specific feature bits. */
2235 void vring_transport_features(struct virtio_device *vdev)
2236 {
2237         unsigned int i;
2238
2239         for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2240                 switch (i) {
2241                 case VIRTIO_RING_F_INDIRECT_DESC:
2242                         break;
2243                 case VIRTIO_RING_F_EVENT_IDX:
2244                         break;
2245                 case VIRTIO_F_VERSION_1:
2246                         break;
2247                 case VIRTIO_F_ACCESS_PLATFORM:
2248                         break;
2249                 case VIRTIO_F_RING_PACKED:
2250                         break;
2251                 case VIRTIO_F_ORDER_PLATFORM:
2252                         break;
2253                 default:
2254                         /* We don't understand this bit. */
2255                         __virtio_clear_bit(vdev, i);
2256                 }
2257         }
2258 }
2259 EXPORT_SYMBOL_GPL(vring_transport_features);
2260
2261 /**
2262  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2263  * @_vq: the struct virtqueue containing the vring of interest.
2264  *
2265  * Returns the size of the vring.  This is mainly used for boasting to
2266  * userspace.  Unlike other operations, this need not be serialized.
2267  */
2268 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2269 {
2270
2271         struct vring_virtqueue *vq = to_vvq(_vq);
2272
2273         return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2274 }
2275 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2276
2277 bool virtqueue_is_broken(struct virtqueue *_vq)
2278 {
2279         struct vring_virtqueue *vq = to_vvq(_vq);
2280
2281         return READ_ONCE(vq->broken);
2282 }
2283 EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2284
2285 /*
2286  * This should prevent the device from being used, allowing drivers to
2287  * recover.  You may need to grab appropriate locks to flush.
2288  */
2289 void virtio_break_device(struct virtio_device *dev)
2290 {
2291         struct virtqueue *_vq;
2292
2293         spin_lock(&dev->vqs_list_lock);
2294         list_for_each_entry(_vq, &dev->vqs, list) {
2295                 struct vring_virtqueue *vq = to_vvq(_vq);
2296
2297                 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2298                 WRITE_ONCE(vq->broken, true);
2299         }
2300         spin_unlock(&dev->vqs_list_lock);
2301 }
2302 EXPORT_SYMBOL_GPL(virtio_break_device);
2303
2304 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2305 {
2306         struct vring_virtqueue *vq = to_vvq(_vq);
2307
2308         BUG_ON(!vq->we_own_ring);
2309
2310         if (vq->packed_ring)
2311                 return vq->packed.ring_dma_addr;
2312
2313         return vq->split.queue_dma_addr;
2314 }
2315 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2316
2317 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2318 {
2319         struct vring_virtqueue *vq = to_vvq(_vq);
2320
2321         BUG_ON(!vq->we_own_ring);
2322
2323         if (vq->packed_ring)
2324                 return vq->packed.driver_event_dma_addr;
2325
2326         return vq->split.queue_dma_addr +
2327                 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2328 }
2329 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2330
2331 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2332 {
2333         struct vring_virtqueue *vq = to_vvq(_vq);
2334
2335         BUG_ON(!vq->we_own_ring);
2336
2337         if (vq->packed_ring)
2338                 return vq->packed.device_event_dma_addr;
2339
2340         return vq->split.queue_dma_addr +
2341                 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2342 }
2343 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2344
2345 /* Only available for split ring */
2346 const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2347 {
2348         return &to_vvq(vq)->split.vring;
2349 }
2350 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2351
2352 MODULE_LICENSE("GPL");