drivers/infiniband/hw/qib/qib_user_sdma.c

   1 /*
   2  * Copyright (c) 2007, 2008, 2009 QLogic Corporation. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/mm.h>
  33 #include <linux/types.h>
  34 #include <linux/device.h>
  35 #include <linux/dmapool.h>
  36 #include <linux/slab.h>
  37 #include <linux/list.h>
  38 #include <linux/highmem.h>
  39 #include <linux/io.h>
  40 #include <linux/uio.h>
  41 #include <linux/rbtree.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/delay.h>
  44 #include <linux/overflow.h>
  45
  46 #include "qib.h"
  47 #include "qib_user_sdma.h"
  48
  49 /* minimum size of header */
  50 #define QIB_USER_SDMA_MIN_HEADER_LENGTH 64
  51 /* expected size of headers (for dma_pool) */
  52 #define QIB_USER_SDMA_EXP_HEADER_LENGTH 64
  53 /* attempt to drain the queue for 5secs */
  54 #define QIB_USER_SDMA_DRAIN_TIMEOUT 250
  55
  56 /*
  57  * track how many times a process open this driver.
  58  */
  59 static struct rb_root qib_user_sdma_rb_root = RB_ROOT;
  60
  61 struct qib_user_sdma_rb_node {
  62         struct rb_node node;
  63         int refcount;
  64         pid_t pid;
  65 };
  66
  67 struct qib_user_sdma_pkt {
  68         struct list_head list;  /* list element */
  69
  70         u8  tiddma;             /* if this is NEW tid-sdma */
  71         u8  largepkt;           /* this is large pkt from kmalloc */
  72         u16 frag_size;          /* frag size used by PSM */
  73         u16 index;              /* last header index or push index */
  74         u16 naddr;              /* dimension of addr (1..3) ... */
  75         u16 addrlimit;          /* addr array size */
  76         u16 tidsmidx;           /* current tidsm index */
  77         u16 tidsmcount;         /* tidsm array item count */
  78         u16 payload_size;       /* payload size so far for header */
  79         u32 bytes_togo;         /* bytes for processing */
  80         u32 counter;            /* sdma pkts queued counter for this entry */
  81         struct qib_tid_session_member *tidsm;   /* tid session member array */
  82         struct qib_user_sdma_queue *pq; /* which pq this pkt belongs to */
  83         u64 added;              /* global descq number of entries */
  84
  85         struct {
  86                 u16 offset;                     /* offset for kvaddr, addr */
  87                 u16 length;                     /* length in page */
  88                 u16 first_desc;                 /* first desc */
  89                 u16 last_desc;                  /* last desc */
  90                 u16 put_page;                   /* should we put_page? */
  91                 u16 dma_mapped;                 /* is page dma_mapped? */
  92                 u16 dma_length;                 /* for dma_unmap_page() */
  93                 u16 padding;
  94                 struct page *page;              /* may be NULL (coherent mem) */
  95                 void *kvaddr;                   /* FIXME: only for pio hack */
  96                 dma_addr_t addr;
  97         } addr[4];   /* max pages, any more and we coalesce */
  98 };
  99
 100 struct qib_user_sdma_queue {
 101         /*
 102          * pkts sent to dma engine are queued on this
 103          * list head.  the type of the elements of this
 104          * list are struct qib_user_sdma_pkt...
 105          */
 106         struct list_head sent;
 107
 108         /*
 109          * Because above list will be accessed by both process and
 110          * signal handler, we need a spinlock for it.
 111          */
 112         spinlock_t sent_lock ____cacheline_aligned_in_smp;
 113
 114         /* headers with expected length are allocated from here... */
 115         char header_cache_name[64];
 116         struct dma_pool *header_cache;
 117
 118         /* packets are allocated from the slab cache... */
 119         char pkt_slab_name[64];
 120         struct kmem_cache *pkt_slab;
 121
 122         /* as packets go on the queued queue, they are counted... */
 123         u32 counter;
 124         u32 sent_counter;
 125         /* pending packets, not sending yet */
 126         u32 num_pending;
 127         /* sending packets, not complete yet */
 128         u32 num_sending;
 129         /* global descq number of entry of last sending packet */
 130         u64 added;
 131
 132         /* dma page table */
 133         struct rb_root dma_pages_root;
 134
 135         struct qib_user_sdma_rb_node *sdma_rb_node;
 136
 137         /* protect everything above... */
 138         struct mutex lock;
 139 };
 140
 141 static struct qib_user_sdma_rb_node *
 142 qib_user_sdma_rb_search(struct rb_root *root, pid_t pid)
 143 {
 144         struct qib_user_sdma_rb_node *sdma_rb_node;
 145         struct rb_node *node = root->rb_node;
 146
 147         while (node) {
 148                 sdma_rb_node = rb_entry(node, struct qib_user_sdma_rb_node,
 149                                         node);
 150                 if (pid < sdma_rb_node->pid)
 151                         node = node->rb_left;
 152                 else if (pid > sdma_rb_node->pid)
 153                         node = node->rb_right;
 154                 else
 155                         return sdma_rb_node;
 156         }
 157         return NULL;
 158 }
 159
 160 static int
 161 qib_user_sdma_rb_insert(struct rb_root *root, struct qib_user_sdma_rb_node *new)
 162 {
 163         struct rb_node **node = &(root->rb_node);
 164         struct rb_node *parent = NULL;
 165         struct qib_user_sdma_rb_node *got;
 166
 167         while (*node) {
 168                 got = rb_entry(*node, struct qib_user_sdma_rb_node, node);
 169                 parent = *node;
 170                 if (new->pid < got->pid)
 171                         node = &((*node)->rb_left);
 172                 else if (new->pid > got->pid)
 173                         node = &((*node)->rb_right);
 174                 else
 175                         return 0;
 176         }
 177
 178         rb_link_node(&new->node, parent, node);
 179         rb_insert_color(&new->node, root);
 180         return 1;
 181 }
 182
 183 struct qib_user_sdma_queue *
 184 qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
 185 {
 186         struct qib_user_sdma_queue *pq =
 187                 kmalloc(sizeof(struct qib_user_sdma_queue), GFP_KERNEL);
 188         struct qib_user_sdma_rb_node *sdma_rb_node;
 189
 190         if (!pq)
 191                 goto done;
 192
 193         pq->counter = 0;
 194         pq->sent_counter = 0;
 195         pq->num_pending = 0;
 196         pq->num_sending = 0;
 197         pq->added = 0;
 198         pq->sdma_rb_node = NULL;
 199
 200         INIT_LIST_HEAD(&pq->sent);
 201         spin_lock_init(&pq->sent_lock);
 202         mutex_init(&pq->lock);
 203
 204         snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
 205                  "qib-user-sdma-pkts-%u-%02u.%02u", unit, ctxt, sctxt);
 206         pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name,
 207                                          sizeof(struct qib_user_sdma_pkt),
 208                                          0, 0, NULL);
 209
 210         if (!pq->pkt_slab)
 211                 goto err_kfree;
 212
 213         snprintf(pq->header_cache_name, sizeof(pq->header_cache_name),
 214                  "qib-user-sdma-headers-%u-%02u.%02u", unit, ctxt, sctxt);
 215         pq->header_cache = dma_pool_create(pq->header_cache_name,
 216                                            dev,
 217                                            QIB_USER_SDMA_EXP_HEADER_LENGTH,
 218                                            4, 0);
 219         if (!pq->header_cache)
 220                 goto err_slab;
 221
 222         pq->dma_pages_root = RB_ROOT;
 223
 224         sdma_rb_node = qib_user_sdma_rb_search(&qib_user_sdma_rb_root,
 225                                         current->pid);
 226         if (sdma_rb_node) {
 227                 sdma_rb_node->refcount++;
 228         } else {
 229                 int ret;
 230
 231                 sdma_rb_node = kmalloc(sizeof(
 232                         struct qib_user_sdma_rb_node), GFP_KERNEL);
 233                 if (!sdma_rb_node)
 234                         goto err_rb;
 235
 236                 sdma_rb_node->refcount = 1;
 237                 sdma_rb_node->pid = current->pid;
 238
 239                 ret = qib_user_sdma_rb_insert(&qib_user_sdma_rb_root,
 240                                         sdma_rb_node);
 241                 BUG_ON(ret == 0);
 242         }
 243         pq->sdma_rb_node = sdma_rb_node;
 244
 245         goto done;
 246
 247 err_rb:
 248         dma_pool_destroy(pq->header_cache);
 249 err_slab:
 250         kmem_cache_destroy(pq->pkt_slab);
 251 err_kfree:
 252         kfree(pq);
 253         pq = NULL;
 254
 255 done:
 256         return pq;
 257 }
 258
 259 static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt,
 260                                     int i, u16 offset, u16 len,
 261                                     u16 first_desc, u16 last_desc,
 262                                     u16 put_page, u16 dma_mapped,
 263                                     struct page *page, void *kvaddr,
 264                                     dma_addr_t dma_addr, u16 dma_length)
 265 {
 266         pkt->addr[i].offset = offset;
 267         pkt->addr[i].length = len;
 268         pkt->addr[i].first_desc = first_desc;
 269         pkt->addr[i].last_desc = last_desc;
 270         pkt->addr[i].put_page = put_page;
 271         pkt->addr[i].dma_mapped = dma_mapped;
 272         pkt->addr[i].page = page;
 273         pkt->addr[i].kvaddr = kvaddr;
 274         pkt->addr[i].addr = dma_addr;
 275         pkt->addr[i].dma_length = dma_length;
 276 }
 277
 278 static void *qib_user_sdma_alloc_header(struct qib_user_sdma_queue *pq,
 279                                 size_t len, dma_addr_t *dma_addr)
 280 {
 281         void *hdr;
 282
 283         if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH)
 284                 hdr = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
 285                                              dma_addr);
 286         else
 287                 hdr = NULL;
 288
 289         if (!hdr) {
 290                 hdr = kmalloc(len, GFP_KERNEL);
 291                 if (!hdr)
 292                         return NULL;
 293
 294                 *dma_addr = 0;
 295         }
 296
 297         return hdr;
 298 }
 299
 300 static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
 301                                        struct qib_user_sdma_queue *pq,
 302                                        struct qib_user_sdma_pkt *pkt,
 303                                        struct page *page, u16 put,
 304                                        u16 offset, u16 len, void *kvaddr)
 305 {
 306         __le16 *pbc16;
 307         void *pbcvaddr;
 308         struct qib_message_header *hdr;
 309         u16 newlen, pbclen, lastdesc, dma_mapped;
 310         u32 vcto;
 311         union qib_seqnum seqnum;
 312         dma_addr_t pbcdaddr;
 313         dma_addr_t dma_addr =
 314                 dma_map_page(&dd->pcidev->dev,
 315                         page, offset, len, DMA_TO_DEVICE);
 316         int ret = 0;
 317
 318         if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
 319                 /*
 320                  * dma mapping error, pkt has not managed
 321                  * this page yet, return the page here so
 322                  * the caller can ignore this page.
 323                  */
 324                 if (put) {
 325                         put_page(page);
 326                 } else {
 327                         /* coalesce case */
 328                         kunmap(page);
 329                         __free_page(page);
 330                 }
 331                 ret = -ENOMEM;
 332                 goto done;
 333         }
 334         offset = 0;
 335         dma_mapped = 1;
 336
 337
 338 next_fragment:
 339
 340         /*
 341          * In tid-sdma, the transfer length is restricted by
 342          * receiver side current tid page length.
 343          */
 344         if (pkt->tiddma && len > pkt->tidsm[pkt->tidsmidx].length)
 345                 newlen = pkt->tidsm[pkt->tidsmidx].length;
 346         else
 347                 newlen = len;
 348
 349         /*
 350          * Then the transfer length is restricted by MTU.
 351          * the last descriptor flag is determined by:
 352          * 1. the current packet is at frag size length.
 353          * 2. the current tid page is done if tid-sdma.
 354          * 3. there is no more byte togo if sdma.
 355          */
 356         lastdesc = 0;
 357         if ((pkt->payload_size + newlen) >= pkt->frag_size) {
 358                 newlen = pkt->frag_size - pkt->payload_size;
 359                 lastdesc = 1;
 360         } else if (pkt->tiddma) {
 361                 if (newlen == pkt->tidsm[pkt->tidsmidx].length)
 362                         lastdesc = 1;
 363         } else {
 364                 if (newlen == pkt->bytes_togo)
 365                         lastdesc = 1;
 366         }
 367
 368         /* fill the next fragment in this page */
 369         qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
 370                 offset, newlen,         /* offset, len */
 371                 0, lastdesc,            /* first last desc */
 372                 put, dma_mapped,        /* put page, dma mapped */
 373                 page, kvaddr,           /* struct page, virt addr */
 374                 dma_addr, len);         /* dma addr, dma length */
 375         pkt->bytes_togo -= newlen;
 376         pkt->payload_size += newlen;
 377         pkt->naddr++;
 378         if (pkt->naddr == pkt->addrlimit) {
 379                 ret = -EFAULT;
 380                 goto done;
 381         }
 382
 383         /* If there is no more byte togo. (lastdesc==1) */
 384         if (pkt->bytes_togo == 0) {
 385                 /* The packet is done, header is not dma mapped yet.
 386                  * it should be from kmalloc */
 387                 if (!pkt->addr[pkt->index].addr) {
 388                         pkt->addr[pkt->index].addr =
 389                                 dma_map_single(&dd->pcidev->dev,
 390                                         pkt->addr[pkt->index].kvaddr,
 391                                         pkt->addr[pkt->index].dma_length,
 392                                         DMA_TO_DEVICE);
 393                         if (dma_mapping_error(&dd->pcidev->dev,
 394                                         pkt->addr[pkt->index].addr)) {
 395                                 ret = -ENOMEM;
 396                                 goto done;
 397                         }
 398                         pkt->addr[pkt->index].dma_mapped = 1;
 399                 }
 400
 401                 goto done;
 402         }
 403
 404         /* If tid-sdma, advance tid info. */
 405         if (pkt->tiddma) {
 406                 pkt->tidsm[pkt->tidsmidx].length -= newlen;
 407                 if (pkt->tidsm[pkt->tidsmidx].length) {
 408                         pkt->tidsm[pkt->tidsmidx].offset += newlen;
 409                 } else {
 410                         pkt->tidsmidx++;
 411                         if (pkt->tidsmidx == pkt->tidsmcount) {
 412                                 ret = -EFAULT;
 413                                 goto done;
 414                         }
 415                 }
 416         }
 417
 418         /*
 419          * If this is NOT the last descriptor. (newlen==len)
 420          * the current packet is not done yet, but the current
 421          * send side page is done.
 422          */
 423         if (lastdesc == 0)
 424                 goto done;
 425
 426         /*
 427          * If running this driver under PSM with message size
 428          * fitting into one transfer unit, it is not possible
 429          * to pass this line. otherwise, it is a buggggg.
 430          */
 431
 432         /*
 433          * Since the current packet is done, and there are more
 434          * bytes togo, we need to create a new sdma header, copying
 435          * from previous sdma header and modify both.
 436          */
 437         pbclen = pkt->addr[pkt->index].length;
 438         pbcvaddr = qib_user_sdma_alloc_header(pq, pbclen, &pbcdaddr);
 439         if (!pbcvaddr) {
 440                 ret = -ENOMEM;
 441                 goto done;
 442         }
 443         /* Copy the previous sdma header to new sdma header */
 444         pbc16 = (__le16 *)pkt->addr[pkt->index].kvaddr;
 445         memcpy(pbcvaddr, pbc16, pbclen);
 446
 447         /* Modify the previous sdma header */
 448         hdr = (struct qib_message_header *)&pbc16[4];
 449
 450         /* New pbc length */
 451         pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->bytes_togo>>2));
 452
 453         /* New packet length */
 454         hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
 455
 456         if (pkt->tiddma) {
 457                 /* turn on the header suppression */
 458                 hdr->iph.pkt_flags =
 459                         cpu_to_le16(le16_to_cpu(hdr->iph.pkt_flags)|0x2);
 460                 /* turn off ACK_REQ: 0x04 and EXPECTED_DONE: 0x20 */
 461                 hdr->flags &= ~(0x04|0x20);
 462         } else {
 463                 /* turn off extra bytes: 20-21 bits */
 464                 hdr->bth[0] = cpu_to_be32(be32_to_cpu(hdr->bth[0])&0xFFCFFFFF);
 465                 /* turn off ACK_REQ: 0x04 */
 466                 hdr->flags &= ~(0x04);
 467         }
 468
 469         /* New kdeth checksum */
 470         vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
 471         hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
 472                 be16_to_cpu(hdr->lrh[2]) -
 473                 ((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
 474                 le16_to_cpu(hdr->iph.pkt_flags));
 475
 476         /* The packet is done, header is not dma mapped yet.
 477          * it should be from kmalloc */
 478         if (!pkt->addr[pkt->index].addr) {
 479                 pkt->addr[pkt->index].addr =
 480                         dma_map_single(&dd->pcidev->dev,
 481                                 pkt->addr[pkt->index].kvaddr,
 482                                 pkt->addr[pkt->index].dma_length,
 483                                 DMA_TO_DEVICE);
 484                 if (dma_mapping_error(&dd->pcidev->dev,
 485                                 pkt->addr[pkt->index].addr)) {
 486                         ret = -ENOMEM;
 487                         goto done;
 488                 }
 489                 pkt->addr[pkt->index].dma_mapped = 1;
 490         }
 491
 492         /* Modify the new sdma header */
 493         pbc16 = (__le16 *)pbcvaddr;
 494         hdr = (struct qib_message_header *)&pbc16[4];
 495
 496         /* New pbc length */
 497         pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->payload_size>>2));
 498
 499         /* New packet length */
 500         hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
 501
 502         if (pkt->tiddma) {
 503                 /* Set new tid and offset for new sdma header */
 504                 hdr->iph.ver_ctxt_tid_offset = cpu_to_le32(
 505                         (le32_to_cpu(hdr->iph.ver_ctxt_tid_offset)&0xFF000000) +
 506                         (pkt->tidsm[pkt->tidsmidx].tid<<QLOGIC_IB_I_TID_SHIFT) +
 507                         (pkt->tidsm[pkt->tidsmidx].offset>>2));
 508         } else {
 509                 /* Middle protocol new packet offset */
 510                 hdr->uwords[2] += pkt->payload_size;
 511         }
 512
 513         /* New kdeth checksum */
 514         vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
 515         hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
 516                 be16_to_cpu(hdr->lrh[2]) -
 517                 ((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
 518                 le16_to_cpu(hdr->iph.pkt_flags));
 519
 520         /* Next sequence number in new sdma header */
 521         seqnum.val = be32_to_cpu(hdr->bth[2]);
 522         if (pkt->tiddma)
 523                 seqnum.seq++;
 524         else
 525                 seqnum.pkt++;
 526         hdr->bth[2] = cpu_to_be32(seqnum.val);
 527
 528         /* Init new sdma header. */
 529         qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
 530                 0, pbclen,              /* offset, len */
 531                 1, 0,                   /* first last desc */
 532                 0, 0,                   /* put page, dma mapped */
 533                 NULL, pbcvaddr,         /* struct page, virt addr */
 534                 pbcdaddr, pbclen);      /* dma addr, dma length */
 535         pkt->index = pkt->naddr;
 536         pkt->payload_size = 0;
 537         pkt->naddr++;
 538         if (pkt->naddr == pkt->addrlimit) {
 539                 ret = -EFAULT;
 540                 goto done;
 541         }
 542
 543         /* Prepare for next fragment in this page */
 544         if (newlen != len) {
 545                 if (dma_mapped) {
 546                         put = 0;
 547                         dma_mapped = 0;
 548                         page = NULL;
 549                         kvaddr = NULL;
 550                 }
 551                 len -= newlen;
 552                 offset += newlen;
 553
 554                 goto next_fragment;
 555         }
 556
 557 done:
 558         return ret;
 559 }
 560
 561 /* we've too many pages in the iovec, coalesce to a single page */
 562 static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
 563                                   struct qib_user_sdma_queue *pq,
 564                                   struct qib_user_sdma_pkt *pkt,
 565                                   const struct iovec *iov,
 566                                   unsigned long niov)
 567 {
 568         int ret = 0;
 569         struct page *page = alloc_page(GFP_KERNEL);
 570         void *mpage_save;
 571         char *mpage;
 572         int i;
 573         int len = 0;
 574
 575         if (!page) {
 576                 ret = -ENOMEM;
 577                 goto done;
 578         }
 579
 580         mpage = kmap(page);
 581         mpage_save = mpage;
 582         for (i = 0; i < niov; i++) {
 583                 int cfur;
 584
 585                 cfur = copy_from_user(mpage,
 586                                       iov[i].iov_base, iov[i].iov_len);
 587                 if (cfur) {
 588                         ret = -EFAULT;
 589                         goto free_unmap;
 590                 }
 591
 592                 mpage += iov[i].iov_len;
 593                 len += iov[i].iov_len;
 594         }
 595
 596         ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
 597                         page, 0, 0, len, mpage_save);
 598         goto done;
 599
 600 free_unmap:
 601         kunmap(page);
 602         __free_page(page);
 603 done:
 604         return ret;
 605 }
 606
 607 /*
 608  * How many pages in this iovec element?
 609  */
 610 static size_t qib_user_sdma_num_pages(const struct iovec *iov)
 611 {
 612         const unsigned long addr  = (unsigned long) iov->iov_base;
 613         const unsigned long  len  = iov->iov_len;
 614         const unsigned long spage = addr & PAGE_MASK;
 615         const unsigned long epage = (addr + len - 1) & PAGE_MASK;
 616
 617         return 1 + ((epage - spage) >> PAGE_SHIFT);
 618 }
 619
 620 static void qib_user_sdma_free_pkt_frag(struct device *dev,
 621                                         struct qib_user_sdma_queue *pq,
 622                                         struct qib_user_sdma_pkt *pkt,
 623                                         int frag)
 624 {
 625         const int i = frag;
 626
 627         if (pkt->addr[i].page) {
 628                 /* only user data has page */
 629                 if (pkt->addr[i].dma_mapped)
 630                         dma_unmap_page(dev,
 631                                        pkt->addr[i].addr,
 632                                        pkt->addr[i].dma_length,
 633                                        DMA_TO_DEVICE);
 634
 635                 if (pkt->addr[i].kvaddr)
 636                         kunmap(pkt->addr[i].page);
 637
 638                 if (pkt->addr[i].put_page)
 639                         put_page(pkt->addr[i].page);
 640                 else
 641                         __free_page(pkt->addr[i].page);
 642         } else if (pkt->addr[i].kvaddr) {
 643                 /* for headers */
 644                 if (pkt->addr[i].dma_mapped) {
 645                         /* from kmalloc & dma mapped */
 646                         dma_unmap_single(dev,
 647                                        pkt->addr[i].addr,
 648                                        pkt->addr[i].dma_length,
 649                                        DMA_TO_DEVICE);
 650                         kfree(pkt->addr[i].kvaddr);
 651                 } else if (pkt->addr[i].addr) {
 652                         /* free coherent mem from cache... */
 653                         dma_pool_free(pq->header_cache,
 654                               pkt->addr[i].kvaddr, pkt->addr[i].addr);
 655                 } else {
 656                         /* from kmalloc but not dma mapped */
 657                         kfree(pkt->addr[i].kvaddr);
 658                 }
 659         }
 660 }
 661
 662 /* return number of pages pinned... */
 663 static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
 664                                    struct qib_user_sdma_queue *pq,
 665                                    struct qib_user_sdma_pkt *pkt,
 666                                    unsigned long addr, int tlen, size_t npages)
 667 {
 668         struct page *pages[8];
 669         int i, j;
 670         int ret = 0;
 671
 672         while (npages) {
 673                 if (npages > 8)
 674                         j = 8;
 675                 else
 676                         j = npages;
 677
 678                 ret = get_user_pages_fast(addr, j, 0, pages);
 679                 if (ret != j) {
 680                         i = 0;
 681                         j = ret;
 682                         ret = -ENOMEM;
 683                         goto free_pages;
 684                 }
 685
 686                 for (i = 0; i < j; i++) {
 687                         /* map the pages... */
 688                         unsigned long fofs = addr & ~PAGE_MASK;
 689                         int flen = ((fofs + tlen) > PAGE_SIZE) ?
 690                                 (PAGE_SIZE - fofs) : tlen;
 691
 692                         ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
 693                                 pages[i], 1, fofs, flen, NULL);
 694                         if (ret < 0) {
 695                                 /* current page has beed taken
 696                                  * care of inside above call.
 697                                  */
 698                                 i++;
 699                                 goto free_pages;
 700                         }
 701
 702                         addr += flen;
 703                         tlen -= flen;
 704                 }
 705
 706                 npages -= j;
 707         }
 708
 709         goto done;
 710
 711         /* if error, return all pages not managed by pkt */
 712 free_pages:
 713         while (i < j)
 714                 put_page(pages[i++]);
 715
 716 done:
 717         return ret;
 718 }
 719
 720 static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
 721                                  struct qib_user_sdma_queue *pq,
 722                                  struct qib_user_sdma_pkt *pkt,
 723                                  const struct iovec *iov,
 724                                  unsigned long niov)
 725 {
 726         int ret = 0;
 727         unsigned long idx;
 728
 729         for (idx = 0; idx < niov; idx++) {
 730                 const size_t npages = qib_user_sdma_num_pages(iov + idx);
 731                 const unsigned long addr = (unsigned long) iov[idx].iov_base;
 732
 733                 ret = qib_user_sdma_pin_pages(dd, pq, pkt, addr,
 734                                               iov[idx].iov_len, npages);
 735                 if (ret < 0)
 736                         goto free_pkt;
 737         }
 738
 739         goto done;
 740
 741 free_pkt:
 742         /* we need to ignore the first entry here */
 743         for (idx = 1; idx < pkt->naddr; idx++)
 744                 qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
 745
 746         /* need to dma unmap the first entry, this is to restore to
 747          * the original state so that caller can free the memory in
 748          * error condition. Caller does not know if dma mapped or not*/
 749         if (pkt->addr[0].dma_mapped) {
 750                 dma_unmap_single(&dd->pcidev->dev,
 751                        pkt->addr[0].addr,
 752                        pkt->addr[0].dma_length,
 753                        DMA_TO_DEVICE);
 754                 pkt->addr[0].addr = 0;
 755                 pkt->addr[0].dma_mapped = 0;
 756         }
 757
 758 done:
 759         return ret;
 760 }
 761
 762 static int qib_user_sdma_init_payload(const struct qib_devdata *dd,
 763                                       struct qib_user_sdma_queue *pq,
 764                                       struct qib_user_sdma_pkt *pkt,
 765                                       const struct iovec *iov,
 766                                       unsigned long niov, int npages)
 767 {
 768         int ret = 0;
 769
 770         if (pkt->frag_size == pkt->bytes_togo &&
 771                         npages >= ARRAY_SIZE(pkt->addr))
 772                 ret = qib_user_sdma_coalesce(dd, pq, pkt, iov, niov);
 773         else
 774                 ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
 775
 776         return ret;
 777 }
 778
 779 /* free a packet list -- return counter value of last packet */
 780 static void qib_user_sdma_free_pkt_list(struct device *dev,
 781                                         struct qib_user_sdma_queue *pq,
 782                                         struct list_head *list)
 783 {
 784         struct qib_user_sdma_pkt *pkt, *pkt_next;
 785
 786         list_for_each_entry_safe(pkt, pkt_next, list, list) {
 787                 int i;
 788
 789                 for (i = 0; i < pkt->naddr; i++)
 790                         qib_user_sdma_free_pkt_frag(dev, pq, pkt, i);
 791
 792                 if (pkt->largepkt)
 793                         kfree(pkt);
 794                 else
 795                         kmem_cache_free(pq->pkt_slab, pkt);
 796         }
 797         INIT_LIST_HEAD(list);
 798 }
 799
 800 /*
 801  * copy headers, coalesce etc -- pq->lock must be held
 802  *
 803  * we queue all the packets to list, returning the
 804  * number of bytes total.  list must be empty initially,
 805  * as, if there is an error we clean it...
 806  */
 807 static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
 808                                     struct qib_pportdata *ppd,
 809                                     struct qib_user_sdma_queue *pq,
 810                                     const struct iovec *iov,
 811                                     unsigned long niov,
 812                                     struct list_head *list,
 813                                     int *maxpkts, int *ndesc)
 814 {
 815         unsigned long idx = 0;
 816         int ret = 0;
 817         int npkts = 0;
 818         __le32 *pbc;
 819         dma_addr_t dma_addr;
 820         struct qib_user_sdma_pkt *pkt = NULL;
 821         size_t len;
 822         size_t nw;
 823         u32 counter = pq->counter;
 824         u16 frag_size;
 825
 826         while (idx < niov && npkts < *maxpkts) {
 827                 const unsigned long addr = (unsigned long) iov[idx].iov_base;
 828                 const unsigned long idx_save = idx;
 829                 unsigned pktnw;
 830                 unsigned pktnwc;
 831                 int nfrags = 0;
 832                 size_t npages = 0;
 833                 size_t bytes_togo = 0;
 834                 int tiddma = 0;
 835                 int cfur;
 836
 837                 len = iov[idx].iov_len;
 838                 nw = len >> 2;
 839
 840                 if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH ||
 841                     len > PAGE_SIZE || len & 3 || addr & 3) {
 842                         ret = -EINVAL;
 843                         goto free_list;
 844                 }
 845
 846                 pbc = qib_user_sdma_alloc_header(pq, len, &dma_addr);
 847                 if (!pbc) {
 848                         ret = -ENOMEM;
 849                         goto free_list;
 850                 }
 851
 852                 cfur = copy_from_user(pbc, iov[idx].iov_base, len);
 853                 if (cfur) {
 854                         ret = -EFAULT;
 855                         goto free_pbc;
 856                 }
 857
 858                 /*
 859                  * This assignment is a bit strange.  it's because the
 860                  * the pbc counts the number of 32 bit words in the full
 861                  * packet _except_ the first word of the pbc itself...
 862                  */
 863                 pktnwc = nw - 1;
 864
 865                 /*
 866                  * pktnw computation yields the number of 32 bit words
 867                  * that the caller has indicated in the PBC.  note that
 868                  * this is one less than the total number of words that
 869                  * goes to the send DMA engine as the first 32 bit word
 870                  * of the PBC itself is not counted.  Armed with this count,
 871                  * we can verify that the packet is consistent with the
 872                  * iovec lengths.
 873                  */
 874                 pktnw = le32_to_cpu(*pbc) & 0xFFFF;
 875                 if (pktnw < pktnwc) {
 876                         ret = -EINVAL;
 877                         goto free_pbc;
 878                 }
 879
 880                 idx++;
 881                 while (pktnwc < pktnw && idx < niov) {
 882                         const size_t slen = iov[idx].iov_len;
 883                         const unsigned long faddr =
 884                                 (unsigned long) iov[idx].iov_base;
 885
 886                         if (slen & 3 || faddr & 3 || !slen) {
 887                                 ret = -EINVAL;
 888                                 goto free_pbc;
 889                         }
 890
 891                         npages += qib_user_sdma_num_pages(&iov[idx]);
 892
 893                         if (check_add_overflow(bytes_togo, slen, &bytes_togo) ||
 894                             bytes_togo > type_max(typeof(pkt->bytes_togo))) {
 895                                 ret = -EINVAL;
 896                                 goto free_pbc;
 897                         }
 898                         pktnwc += slen >> 2;
 899                         idx++;
 900                         nfrags++;
 901                 }
 902
 903                 if (pktnwc != pktnw) {
 904                         ret = -EINVAL;
 905                         goto free_pbc;
 906                 }
 907
 908                 frag_size = ((le32_to_cpu(*pbc))>>16) & 0xFFFF;
 909                 if (((frag_size ? frag_size : bytes_togo) + len) >
 910                                                 ppd->ibmaxlen) {
 911                         ret = -EINVAL;
 912                         goto free_pbc;
 913                 }
 914
 915                 if (frag_size) {
 916                         size_t tidsmsize, n, pktsize, sz, addrlimit;
 917
 918                         n = npages*((2*PAGE_SIZE/frag_size)+1);
 919                         pktsize = struct_size(pkt, addr, n);
 920
 921                         /*
 922                          * Determine if this is tid-sdma or just sdma.
 923                          */
 924                         tiddma = (((le32_to_cpu(pbc[7])>>
 925                                 QLOGIC_IB_I_TID_SHIFT)&
 926                                 QLOGIC_IB_I_TID_MASK) !=
 927                                 QLOGIC_IB_I_TID_MASK);
 928
 929                         if (tiddma)
 930                                 tidsmsize = iov[idx].iov_len;
 931                         else
 932                                 tidsmsize = 0;
 933
 934                         if (check_add_overflow(pktsize, tidsmsize, &sz)) {
 935                                 ret = -EINVAL;
 936                                 goto free_pbc;
 937                         }
 938                         pkt = kmalloc(sz, GFP_KERNEL);
 939                         if (!pkt) {
 940                                 ret = -ENOMEM;
 941                                 goto free_pbc;
 942                         }
 943                         pkt->largepkt = 1;
 944                         pkt->frag_size = frag_size;
 945                         if (check_add_overflow(n, ARRAY_SIZE(pkt->addr),
 946                                                &addrlimit) ||
 947                             addrlimit > type_max(typeof(pkt->addrlimit))) {
 948                                 ret = -EINVAL;
 949                                 goto free_pkt;
 950                         }
 951                         pkt->addrlimit = addrlimit;
 952
 953                         if (tiddma) {
 954                                 char *tidsm = (char *)pkt + pktsize;
 955
 956                                 cfur = copy_from_user(tidsm,
 957                                         iov[idx].iov_base, tidsmsize);
 958                                 if (cfur) {
 959                                         ret = -EFAULT;
 960                                         goto free_pkt;
 961                                 }
 962                                 pkt->tidsm =
 963                                         (struct qib_tid_session_member *)tidsm;
 964                                 pkt->tidsmcount = tidsmsize/
 965                                         sizeof(struct qib_tid_session_member);
 966                                 pkt->tidsmidx = 0;
 967                                 idx++;
 968                         }
 969
 970                         /*
 971                          * pbc 'fill1' field is borrowed to pass frag size,
 972                          * we need to clear it after picking frag size, the
 973                          * hardware requires this field to be zero.
 974                          */
 975                         *pbc = cpu_to_le32(le32_to_cpu(*pbc) & 0x0000FFFF);
 976                 } else {
 977                         pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
 978                         if (!pkt) {
 979                                 ret = -ENOMEM;
 980                                 goto free_pbc;
 981                         }
 982                         pkt->largepkt = 0;
 983                         pkt->frag_size = bytes_togo;
 984                         pkt->addrlimit = ARRAY_SIZE(pkt->addr);
 985                 }
 986                 pkt->bytes_togo = bytes_togo;
 987                 pkt->payload_size = 0;
 988                 pkt->counter = counter;
 989                 pkt->tiddma = tiddma;
 990
 991                 /* setup the first header */
 992                 qib_user_sdma_init_frag(pkt, 0, /* index */
 993                         0, len,         /* offset, len */
 994                         1, 0,           /* first last desc */
 995                         0, 0,           /* put page, dma mapped */
 996                         NULL, pbc,      /* struct page, virt addr */
 997                         dma_addr, len); /* dma addr, dma length */
 998                 pkt->index = 0;
 999                 pkt->naddr = 1;
1000
1001                 if (nfrags) {
1002                         ret = qib_user_sdma_init_payload(dd, pq, pkt,
1003                                                          iov + idx_save + 1,
1004                                                          nfrags, npages);
1005                         if (ret < 0)
1006                                 goto free_pkt;
1007                 } else {
1008                         /* since there is no payload, mark the
1009                          * header as the last desc. */
1010                         pkt->addr[0].last_desc = 1;
1011
1012                         if (dma_addr == 0) {
1013                                 /*
1014                                  * the header is not dma mapped yet.
1015                                  * it should be from kmalloc.
1016                                  */
1017                                 dma_addr = dma_map_single(&dd->pcidev->dev,
1018                                         pbc, len, DMA_TO_DEVICE);
1019                                 if (dma_mapping_error(&dd->pcidev->dev,
1020                                                                 dma_addr)) {
1021                                         ret = -ENOMEM;
1022                                         goto free_pkt;
1023                                 }
1024                                 pkt->addr[0].addr = dma_addr;
1025                                 pkt->addr[0].dma_mapped = 1;
1026                         }
1027                 }
1028
1029                 counter++;
1030                 npkts++;
1031                 pkt->pq = pq;
1032                 pkt->index = 0; /* reset index for push on hw */
1033                 *ndesc += pkt->naddr;
1034
1035                 list_add_tail(&pkt->list, list);
1036         }
1037
1038         *maxpkts = npkts;
1039         ret = idx;
1040         goto done;
1041
1042 free_pkt:
1043         if (pkt->largepkt)
1044                 kfree(pkt);
1045         else
1046                 kmem_cache_free(pq->pkt_slab, pkt);
1047 free_pbc:
1048         if (dma_addr)
1049                 dma_pool_free(pq->header_cache, pbc, dma_addr);
1050         else
1051                 kfree(pbc);
1052 free_list:
1053         qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
1054 done:
1055         return ret;
1056 }
1057
1058 static void qib_user_sdma_set_complete_counter(struct qib_user_sdma_queue *pq,
1059                                                u32 c)
1060 {
1061         pq->sent_counter = c;
1062 }
1063
1064 /* try to clean out queue -- needs pq->lock */
1065 static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
1066                                      struct qib_user_sdma_queue *pq)
1067 {
1068         struct qib_devdata *dd = ppd->dd;
1069         struct list_head free_list;
1070         struct qib_user_sdma_pkt *pkt;
1071         struct qib_user_sdma_pkt *pkt_prev;
1072         unsigned long flags;
1073         int ret = 0;
1074
1075         if (!pq->num_sending)
1076                 return 0;
1077
1078         INIT_LIST_HEAD(&free_list);
1079
1080         /*
1081          * We need this spin lock here because interrupt handler
1082          * might modify this list in qib_user_sdma_send_desc(), also
1083          * we can not get interrupted, otherwise it is a deadlock.
1084          */
1085         spin_lock_irqsave(&pq->sent_lock, flags);
1086         list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
1087                 s64 descd = ppd->sdma_descq_removed - pkt->added;
1088
1089                 if (descd < 0)
1090                         break;
1091
1092                 list_move_tail(&pkt->list, &free_list);
1093
1094                 /* one more packet cleaned */
1095                 ret++;
1096                 pq->num_sending--;
1097         }
1098         spin_unlock_irqrestore(&pq->sent_lock, flags);
1099
1100         if (!list_empty(&free_list)) {
1101                 u32 counter;
1102
1103                 pkt = list_entry(free_list.prev,
1104                                  struct qib_user_sdma_pkt, list);
1105                 counter = pkt->counter;
1106
1107                 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1108                 qib_user_sdma_set_complete_counter(pq, counter);
1109         }
1110
1111         return ret;
1112 }
1113
1114 void qib_user_sdma_queue_destroy(struct qib_user_sdma_queue *pq)
1115 {
1116         if (!pq)
1117                 return;
1118
1119         pq->sdma_rb_node->refcount--;
1120         if (pq->sdma_rb_node->refcount == 0) {
1121                 rb_erase(&pq->sdma_rb_node->node, &qib_user_sdma_rb_root);
1122                 kfree(pq->sdma_rb_node);
1123         }
1124         dma_pool_destroy(pq->header_cache);
1125         kmem_cache_destroy(pq->pkt_slab);
1126         kfree(pq);
1127 }
1128
1129 /* clean descriptor queue, returns > 0 if some elements cleaned */
1130 static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd)
1131 {
1132         int ret;
1133         unsigned long flags;
1134
1135         spin_lock_irqsave(&ppd->sdma_lock, flags);
1136         ret = qib_sdma_make_progress(ppd);
1137         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1138
1139         return ret;
1140 }
1141
1142 /* we're in close, drain packets so that we can cleanup successfully... */
1143 void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
1144                                struct qib_user_sdma_queue *pq)
1145 {
1146         struct qib_devdata *dd = ppd->dd;
1147         unsigned long flags;
1148         int i;
1149
1150         if (!pq)
1151                 return;
1152
1153         for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) {
1154                 mutex_lock(&pq->lock);
1155                 if (!pq->num_pending && !pq->num_sending) {
1156                         mutex_unlock(&pq->lock);
1157                         break;
1158                 }
1159                 qib_user_sdma_hwqueue_clean(ppd);
1160                 qib_user_sdma_queue_clean(ppd, pq);
1161                 mutex_unlock(&pq->lock);
1162                 msleep(20);
1163         }
1164
1165         if (pq->num_pending || pq->num_sending) {
1166                 struct qib_user_sdma_pkt *pkt;
1167                 struct qib_user_sdma_pkt *pkt_prev;
1168                 struct list_head free_list;
1169
1170                 mutex_lock(&pq->lock);
1171                 spin_lock_irqsave(&ppd->sdma_lock, flags);
1172                 /*
1173                  * Since we hold sdma_lock, it is safe without sent_lock.
1174                  */
1175                 if (pq->num_pending) {
1176                         list_for_each_entry_safe(pkt, pkt_prev,
1177                                         &ppd->sdma_userpending, list) {
1178                                 if (pkt->pq == pq) {
1179                                         list_move_tail(&pkt->list, &pq->sent);
1180                                         pq->num_pending--;
1181                                         pq->num_sending++;
1182                                 }
1183                         }
1184                 }
1185                 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1186
1187                 qib_dev_err(dd, "user sdma lists not empty: forcing!\n");
1188                 INIT_LIST_HEAD(&free_list);
1189                 list_splice_init(&pq->sent, &free_list);
1190                 pq->num_sending = 0;
1191                 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1192                 mutex_unlock(&pq->lock);
1193         }
1194 }
1195
1196 static inline __le64 qib_sdma_make_desc0(u8 gen,
1197                                          u64 addr, u64 dwlen, u64 dwoffset)
1198 {
1199         return cpu_to_le64(/* SDmaPhyAddr[31:0] */
1200                            ((addr & 0xfffffffcULL) << 32) |
1201                            /* SDmaGeneration[1:0] */
1202                            ((gen & 3ULL) << 30) |
1203                            /* SDmaDwordCount[10:0] */
1204                            ((dwlen & 0x7ffULL) << 16) |
1205                            /* SDmaBufOffset[12:2] */
1206                            (dwoffset & 0x7ffULL));
1207 }
1208
1209 static inline __le64 qib_sdma_make_first_desc0(__le64 descq)
1210 {
1211         return descq | cpu_to_le64(1ULL << 12);
1212 }
1213
1214 static inline __le64 qib_sdma_make_last_desc0(__le64 descq)
1215 {
1216                                               /* last */  /* dma head */
1217         return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13);
1218 }
1219
1220 static inline __le64 qib_sdma_make_desc1(u64 addr)
1221 {
1222         /* SDmaPhyAddr[47:32] */
1223         return cpu_to_le64(addr >> 32);
1224 }
1225
1226 static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
1227                                     struct qib_user_sdma_pkt *pkt, int idx,
1228                                     unsigned ofs, u16 tail, u8 gen)
1229 {
1230         const u64 addr = (u64) pkt->addr[idx].addr +
1231                 (u64) pkt->addr[idx].offset;
1232         const u64 dwlen = (u64) pkt->addr[idx].length / 4;
1233         __le64 *descqp;
1234         __le64 descq0;
1235
1236         descqp = &ppd->sdma_descq[tail].qw[0];
1237
1238         descq0 = qib_sdma_make_desc0(gen, addr, dwlen, ofs);
1239         if (pkt->addr[idx].first_desc)
1240                 descq0 = qib_sdma_make_first_desc0(descq0);
1241         if (pkt->addr[idx].last_desc) {
1242                 descq0 = qib_sdma_make_last_desc0(descq0);
1243                 if (ppd->sdma_intrequest) {
1244                         descq0 |= cpu_to_le64(1ULL << 15);
1245                         ppd->sdma_intrequest = 0;
1246                 }
1247         }
1248
1249         descqp[0] = descq0;
1250         descqp[1] = qib_sdma_make_desc1(addr);
1251 }
1252
1253 void qib_user_sdma_send_desc(struct qib_pportdata *ppd,
1254                                 struct list_head *pktlist)
1255 {
1256         struct qib_devdata *dd = ppd->dd;
1257         u16 nfree, nsent;
1258         u16 tail, tail_c;
1259         u8 gen, gen_c;
1260
1261         nfree = qib_sdma_descq_freecnt(ppd);
1262         if (!nfree)
1263                 return;
1264
1265 retry:
1266         nsent = 0;
1267         tail_c = tail = ppd->sdma_descq_tail;
1268         gen_c = gen = ppd->sdma_generation;
1269         while (!list_empty(pktlist)) {
1270                 struct qib_user_sdma_pkt *pkt =
1271                         list_entry(pktlist->next, struct qib_user_sdma_pkt,
1272                                    list);
1273                 int i, j, c = 0;
1274                 unsigned ofs = 0;
1275                 u16 dtail = tail;
1276
1277                 for (i = pkt->index; i < pkt->naddr && nfree; i++) {
1278                         qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);
1279                         ofs += pkt->addr[i].length >> 2;
1280
1281                         if (++tail == ppd->sdma_descq_cnt) {
1282                                 tail = 0;
1283                                 ++gen;
1284                                 ppd->sdma_intrequest = 1;
1285                         } else if (tail == (ppd->sdma_descq_cnt>>1)) {
1286                                 ppd->sdma_intrequest = 1;
1287                         }
1288                         nfree--;
1289                         if (pkt->addr[i].last_desc == 0)
1290                                 continue;
1291
1292                         /*
1293                          * If the packet is >= 2KB mtu equivalent, we
1294                          * have to use the large buffers, and have to
1295                          * mark each descriptor as part of a large
1296                          * buffer packet.
1297                          */
1298                         if (ofs > dd->piosize2kmax_dwords) {
1299                                 for (j = pkt->index; j <= i; j++) {
1300                                         ppd->sdma_descq[dtail].qw[0] |=
1301                                                 cpu_to_le64(1ULL << 14);
1302                                         if (++dtail == ppd->sdma_descq_cnt)
1303                                                 dtail = 0;
1304                                 }
1305                         }
1306                         c += i + 1 - pkt->index;
1307                         pkt->index = i + 1; /* index for next first */
1308                         tail_c = dtail = tail;
1309                         gen_c = gen;
1310                         ofs = 0;  /* reset for next packet */
1311                 }
1312
1313                 ppd->sdma_descq_added += c;
1314                 nsent += c;
1315                 if (pkt->index == pkt->naddr) {
1316                         pkt->added = ppd->sdma_descq_added;
1317                         pkt->pq->added = pkt->added;
1318                         pkt->pq->num_pending--;
1319                         spin_lock(&pkt->pq->sent_lock);
1320                         pkt->pq->num_sending++;
1321                         list_move_tail(&pkt->list, &pkt->pq->sent);
1322                         spin_unlock(&pkt->pq->sent_lock);
1323                 }
1324                 if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)
1325                         break;
1326         }
1327
1328         /* advance the tail on the chip if necessary */
1329         if (ppd->sdma_descq_tail != tail_c) {
1330                 ppd->sdma_generation = gen_c;
1331                 dd->f_sdma_update_tail(ppd, tail_c);
1332         }
1333
1334         if (nfree && !list_empty(pktlist))
1335                 goto retry;
1336 }
1337
1338 /* pq->lock must be held, get packets on the wire... */
1339 static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,
1340                                  struct qib_user_sdma_queue *pq,
1341                                  struct list_head *pktlist, int count)
1342 {
1343         unsigned long flags;
1344
1345         if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))
1346                 return -ECOMM;
1347
1348         /* non-blocking mode */
1349         if (pq->sdma_rb_node->refcount > 1) {
1350                 spin_lock_irqsave(&ppd->sdma_lock, flags);
1351                 if (unlikely(!__qib_sdma_running(ppd))) {
1352                         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1353                         return -ECOMM;
1354                 }
1355                 pq->num_pending += count;
1356                 list_splice_tail_init(pktlist, &ppd->sdma_userpending);
1357                 qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
1358                 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1359                 return 0;
1360         }
1361
1362         /* In this case, descriptors from this process are not
1363          * linked to ppd pending queue, interrupt handler
1364          * won't update this process, it is OK to directly
1365          * modify without sdma lock.
1366          */
1367
1368
1369         pq->num_pending += count;
1370         /*
1371          * Blocking mode for single rail process, we must
1372          * release/regain sdma_lock to give other process
1373          * chance to make progress. This is important for
1374          * performance.
1375          */
1376         do {
1377                 spin_lock_irqsave(&ppd->sdma_lock, flags);
1378                 if (unlikely(!__qib_sdma_running(ppd))) {
1379                         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1380                         return -ECOMM;
1381                 }
1382                 qib_user_sdma_send_desc(ppd, pktlist);
1383                 if (!list_empty(pktlist))
1384                         qib_sdma_make_progress(ppd);
1385                 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1386         } while (!list_empty(pktlist));
1387
1388         return 0;
1389 }
1390
1391 int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
1392                          struct qib_user_sdma_queue *pq,
1393                          const struct iovec *iov,
1394                          unsigned long dim)
1395 {
1396         struct qib_devdata *dd = rcd->dd;
1397         struct qib_pportdata *ppd = rcd->ppd;
1398         int ret = 0;
1399         struct list_head list;
1400         int npkts = 0;
1401
1402         INIT_LIST_HEAD(&list);
1403
1404         mutex_lock(&pq->lock);
1405
1406         /* why not -ECOMM like qib_user_sdma_push_pkts() below? */
1407         if (!qib_sdma_running(ppd))
1408                 goto done_unlock;
1409
1410         /* if I have packets not complete yet */
1411         if (pq->added > ppd->sdma_descq_removed)
1412                 qib_user_sdma_hwqueue_clean(ppd);
1413         /* if I have complete packets to be freed */
1414         if (pq->num_sending)
1415                 qib_user_sdma_queue_clean(ppd, pq);
1416
1417         while (dim) {
1418                 int mxp = 1;
1419                 int ndesc = 0;
1420
1421                 ret = qib_user_sdma_queue_pkts(dd, ppd, pq,
1422                                 iov, dim, &list, &mxp, &ndesc);
1423                 if (ret < 0)
1424                         goto done_unlock;
1425                 else {
1426                         dim -= ret;
1427                         iov += ret;
1428                 }
1429
1430                 /* force packets onto the sdma hw queue... */
1431                 if (!list_empty(&list)) {
1432                         /*
1433                          * Lazily clean hw queue.
1434                          */
1435                         if (qib_sdma_descq_freecnt(ppd) < ndesc) {
1436                                 qib_user_sdma_hwqueue_clean(ppd);
1437                                 if (pq->num_sending)
1438                                         qib_user_sdma_queue_clean(ppd, pq);
1439                         }
1440
1441                         ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);
1442                         if (ret < 0)
1443                                 goto done_unlock;
1444                         else {
1445                                 npkts += mxp;
1446                                 pq->counter += mxp;
1447                         }
1448                 }
1449         }
1450
1451 done_unlock:
1452         if (!list_empty(&list))
1453                 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list);
1454         mutex_unlock(&pq->lock);
1455
1456         return (ret < 0) ? ret : npkts;
1457 }
1458
1459 int qib_user_sdma_make_progress(struct qib_pportdata *ppd,
1460                                 struct qib_user_sdma_queue *pq)
1461 {
1462         int ret = 0;
1463
1464         mutex_lock(&pq->lock);
1465         qib_user_sdma_hwqueue_clean(ppd);
1466         ret = qib_user_sdma_queue_clean(ppd, pq);
1467         mutex_unlock(&pq->lock);
1468
1469         return ret;
1470 }
1471
1472 u32 qib_user_sdma_complete_counter(const struct qib_user_sdma_queue *pq)
1473 {
1474         return pq ? pq->sent_counter : 0;
1475 }
1476
1477 u32 qib_user_sdma_inflight_counter(struct qib_user_sdma_queue *pq)
1478 {
1479         return pq ? pq->counter : 0;
1480 }