drivers/char/xillybus/xillybus_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/drivers/misc/xillybus_core.c
   4  *
   5  * Copyright 2011 Xillybus Ltd, http://xillybus.com
   6  *
   7  * Driver for the Xillybus FPGA/host framework.
   8  *
   9  * This driver interfaces with a special IP core in an FPGA, setting up
  10  * a pipe between a hardware FIFO in the programmable logic and a device
  11  * file in the host. The number of such pipes and their attributes are
  12  * set up on the logic. This driver detects these automatically and
  13  * creates the device files accordingly.
  14  */
  15
  16 #include <linux/list.h>
  17 #include <linux/device.h>
  18 #include <linux/module.h>
  19 #include <linux/io.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/sched.h>
  23 #include <linux/fs.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/mutex.h>
  26 #include <linux/crc32.h>
  27 #include <linux/poll.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/workqueue.h>
  31 #include "xillybus.h"
  32 #include "xillybus_class.h"
  33
  34 MODULE_DESCRIPTION("Xillybus core functions");
  35 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  36 MODULE_ALIAS("xillybus_core");
  37 MODULE_LICENSE("GPL v2");
  38
  39 /* General timeout is 100 ms, rx timeout is 10 ms */
  40 #define XILLY_RX_TIMEOUT (10*HZ/1000)
  41 #define XILLY_TIMEOUT (100*HZ/1000)
  42
  43 #define fpga_msg_ctrl_reg              0x0008
  44 #define fpga_dma_control_reg           0x0020
  45 #define fpga_dma_bufno_reg             0x0024
  46 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
  47 #define fpga_dma_bufaddr_highaddr_reg  0x002c
  48 #define fpga_buf_ctrl_reg              0x0030
  49 #define fpga_buf_offset_reg            0x0034
  50 #define fpga_endian_reg                0x0040
  51
  52 #define XILLYMSG_OPCODE_RELEASEBUF 1
  53 #define XILLYMSG_OPCODE_QUIESCEACK 2
  54 #define XILLYMSG_OPCODE_FIFOEOF 3
  55 #define XILLYMSG_OPCODE_FATAL_ERROR 4
  56 #define XILLYMSG_OPCODE_NONEMPTY 5
  57
  58 static const char xillyname[] = "xillybus";
  59
  60 static struct workqueue_struct *xillybus_wq;
  61
  62 /*
  63  * Locking scheme: Mutexes protect invocations of character device methods.
  64  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  65  *
  66  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  67  * buffers' end_offset fields against changes made by IRQ handler (and in
  68  * theory, other file request handlers, but the mutex handles that). Nothing
  69  * else.
  70  * They are held for short direct memory manipulations. Needless to say,
  71  * no mutex locking is allowed when a spinlock is held.
  72  *
  73  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  74  *
  75  * register_mutex is endpoint-specific, and is held when non-atomic
  76  * register operations are performed. wr_mutex and rd_mutex may be
  77  * held when register_mutex is taken, but none of the spinlocks. Note that
  78  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  79  * which are unrelated to buf_offset_reg, since they are harmless.
  80  *
  81  * Blocking on the wait queues is allowed with mutexes held, but not with
  82  * spinlocks.
  83  *
  84  * Only interruptible blocking is allowed on mutexes and wait queues.
  85  *
  86  * All in all, the locking order goes (with skips allowed, of course):
  87  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
  88  */
  89
  90 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
  91 {
  92         int opcode;
  93         int msg_channel, msg_bufno, msg_data, msg_dir;
  94
  95         opcode = (buf[0] >> 24) & 0xff;
  96         msg_dir = buf[0] & 1;
  97         msg_channel = (buf[0] >> 1) & 0x7ff;
  98         msg_bufno = (buf[0] >> 12) & 0x3ff;
  99         msg_data = buf[1] & 0xfffffff;
 100
 101         dev_warn(endpoint->dev,
 102                  "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 103                  opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 104 }
 105
 106 /*
 107  * xillybus_isr assumes the interrupt is allocated exclusively to it,
 108  * which is the natural case MSI and several other hardware-oriented
 109  * interrupts. Sharing is not allowed.
 110  */
 111
 112 irqreturn_t xillybus_isr(int irq, void *data)
 113 {
 114         struct xilly_endpoint *ep = data;
 115         u32 *buf;
 116         unsigned int buf_size;
 117         int i;
 118         int opcode;
 119         unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 120         struct xilly_channel *channel;
 121
 122         buf = ep->msgbuf_addr;
 123         buf_size = ep->msg_buf_size/sizeof(u32);
 124
 125         dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr,
 126                                 ep->msg_buf_size, DMA_FROM_DEVICE);
 127
 128         for (i = 0; i < buf_size; i += 2) {
 129                 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 130                         malformed_message(ep, &buf[i]);
 131                         dev_warn(ep->dev,
 132                                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 133                                  ((buf[i+1] >> 28) & 0xf),
 134                                  ep->msg_counter,
 135                                  i/2);
 136
 137                         if (++ep->failed_messages > 10) {
 138                                 dev_err(ep->dev,
 139                                         "Lost sync with interrupt messages. Stopping.\n");
 140                         } else {
 141                                 dma_sync_single_for_device(ep->dev,
 142                                                            ep->msgbuf_dma_addr,
 143                                                            ep->msg_buf_size,
 144                                                            DMA_FROM_DEVICE);
 145
 146                                 iowrite32(0x01,  /* Message NACK */
 147                                           ep->registers + fpga_msg_ctrl_reg);
 148                         }
 149                         return IRQ_HANDLED;
 150                 } else if (buf[i] & (1 << 22)) /* Last message */
 151                         break;
 152         }
 153
 154         if (i >= buf_size) {
 155                 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 156                 return IRQ_HANDLED;
 157         }
 158
 159         buf_size = i + 2;
 160
 161         for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
 162                 opcode = (buf[i] >> 24) & 0xff;
 163
 164                 msg_dir = buf[i] & 1;
 165                 msg_channel = (buf[i] >> 1) & 0x7ff;
 166                 msg_bufno = (buf[i] >> 12) & 0x3ff;
 167                 msg_data = buf[i+1] & 0xfffffff;
 168
 169                 switch (opcode) {
 170                 case XILLYMSG_OPCODE_RELEASEBUF:
 171                         if ((msg_channel > ep->num_channels) ||
 172                             (msg_channel == 0)) {
 173                                 malformed_message(ep, &buf[i]);
 174                                 break;
 175                         }
 176
 177                         channel = ep->channels[msg_channel];
 178
 179                         if (msg_dir) { /* Write channel */
 180                                 if (msg_bufno >= channel->num_wr_buffers) {
 181                                         malformed_message(ep, &buf[i]);
 182                                         break;
 183                                 }
 184                                 spin_lock(&channel->wr_spinlock);
 185                                 channel->wr_buffers[msg_bufno]->end_offset =
 186                                         msg_data;
 187                                 channel->wr_fpga_buf_idx = msg_bufno;
 188                                 channel->wr_empty = 0;
 189                                 channel->wr_sleepy = 0;
 190                                 spin_unlock(&channel->wr_spinlock);
 191
 192                                 wake_up_interruptible(&channel->wr_wait);
 193
 194                         } else {
 195                                 /* Read channel */
 196
 197                                 if (msg_bufno >= channel->num_rd_buffers) {
 198                                         malformed_message(ep, &buf[i]);
 199                                         break;
 200                                 }
 201
 202                                 spin_lock(&channel->rd_spinlock);
 203                                 channel->rd_fpga_buf_idx = msg_bufno;
 204                                 channel->rd_full = 0;
 205                                 spin_unlock(&channel->rd_spinlock);
 206
 207                                 wake_up_interruptible(&channel->rd_wait);
 208                                 if (!channel->rd_synchronous)
 209                                         queue_delayed_work(
 210                                                 xillybus_wq,
 211                                                 &channel->rd_workitem,
 212                                                 XILLY_RX_TIMEOUT);
 213                         }
 214
 215                         break;
 216                 case XILLYMSG_OPCODE_NONEMPTY:
 217                         if ((msg_channel > ep->num_channels) ||
 218                             (msg_channel == 0) || (!msg_dir) ||
 219                             !ep->channels[msg_channel]->wr_supports_nonempty) {
 220                                 malformed_message(ep, &buf[i]);
 221                                 break;
 222                         }
 223
 224                         channel = ep->channels[msg_channel];
 225
 226                         if (msg_bufno >= channel->num_wr_buffers) {
 227                                 malformed_message(ep, &buf[i]);
 228                                 break;
 229                         }
 230                         spin_lock(&channel->wr_spinlock);
 231                         if (msg_bufno == channel->wr_host_buf_idx)
 232                                 channel->wr_ready = 1;
 233                         spin_unlock(&channel->wr_spinlock);
 234
 235                         wake_up_interruptible(&channel->wr_ready_wait);
 236
 237                         break;
 238                 case XILLYMSG_OPCODE_QUIESCEACK:
 239                         ep->idtlen = msg_data;
 240                         wake_up_interruptible(&ep->ep_wait);
 241
 242                         break;
 243                 case XILLYMSG_OPCODE_FIFOEOF:
 244                         if ((msg_channel > ep->num_channels) ||
 245                             (msg_channel == 0) || (!msg_dir) ||
 246                             !ep->channels[msg_channel]->num_wr_buffers) {
 247                                 malformed_message(ep, &buf[i]);
 248                                 break;
 249                         }
 250                         channel = ep->channels[msg_channel];
 251                         spin_lock(&channel->wr_spinlock);
 252                         channel->wr_eof = msg_bufno;
 253                         channel->wr_sleepy = 0;
 254
 255                         channel->wr_hangup = channel->wr_empty &&
 256                                 (channel->wr_host_buf_idx == msg_bufno);
 257
 258                         spin_unlock(&channel->wr_spinlock);
 259
 260                         wake_up_interruptible(&channel->wr_wait);
 261
 262                         break;
 263                 case XILLYMSG_OPCODE_FATAL_ERROR:
 264                         ep->fatal_error = 1;
 265                         wake_up_interruptible(&ep->ep_wait); /* For select() */
 266                         dev_err(ep->dev,
 267                                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 268                         break;
 269                 default:
 270                         malformed_message(ep, &buf[i]);
 271                         break;
 272                 }
 273         }
 274
 275         dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr,
 276                                    ep->msg_buf_size, DMA_FROM_DEVICE);
 277
 278         ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 279         ep->failed_messages = 0;
 280         iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 281
 282         return IRQ_HANDLED;
 283 }
 284 EXPORT_SYMBOL(xillybus_isr);
 285
 286 /*
 287  * A few trivial memory management functions.
 288  * NOTE: These functions are used only on probe and remove, and therefore
 289  * no locks are applied!
 290  */
 291
 292 static void xillybus_autoflush(struct work_struct *work);
 293
 294 struct xilly_alloc_state {
 295         void *salami;
 296         int left_of_salami;
 297         int nbuffer;
 298         enum dma_data_direction direction;
 299         u32 regdirection;
 300 };
 301
 302 static void xilly_unmap(void *ptr)
 303 {
 304         struct xilly_mapping *data = ptr;
 305
 306         dma_unmap_single(data->device, data->dma_addr,
 307                          data->size, data->direction);
 308
 309         kfree(ptr);
 310 }
 311
 312 static int xilly_map_single(struct xilly_endpoint *ep,
 313                             void *ptr,
 314                             size_t size,
 315                             int direction,
 316                             dma_addr_t *ret_dma_handle
 317         )
 318 {
 319         dma_addr_t addr;
 320         struct xilly_mapping *this;
 321
 322         this = kzalloc(sizeof(*this), GFP_KERNEL);
 323         if (!this)
 324                 return -ENOMEM;
 325
 326         addr = dma_map_single(ep->dev, ptr, size, direction);
 327
 328         if (dma_mapping_error(ep->dev, addr)) {
 329                 kfree(this);
 330                 return -ENODEV;
 331         }
 332
 333         this->device = ep->dev;
 334         this->dma_addr = addr;
 335         this->size = size;
 336         this->direction = direction;
 337
 338         *ret_dma_handle = addr;
 339
 340         return devm_add_action_or_reset(ep->dev, xilly_unmap, this);
 341 }
 342
 343 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
 344                                  struct xilly_alloc_state *s,
 345                                  struct xilly_buffer **buffers,
 346                                  int bufnum, int bytebufsize)
 347 {
 348         int i, rc;
 349         dma_addr_t dma_addr;
 350         struct device *dev = ep->dev;
 351         struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 352
 353         if (buffers) { /* Not the message buffer */
 354                 this_buffer = devm_kcalloc(dev, bufnum,
 355                                            sizeof(struct xilly_buffer),
 356                                            GFP_KERNEL);
 357                 if (!this_buffer)
 358                         return -ENOMEM;
 359         }
 360
 361         for (i = 0; i < bufnum; i++) {
 362                 /*
 363                  * Buffers are expected in descending size order, so there
 364                  * is either enough space for this buffer or none at all.
 365                  */
 366
 367                 if ((s->left_of_salami < bytebufsize) &&
 368                     (s->left_of_salami > 0)) {
 369                         dev_err(ep->dev,
 370                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 371                         return -ENODEV;
 372                 }
 373
 374                 if (s->left_of_salami == 0) {
 375                         int allocorder, allocsize;
 376
 377                         allocsize = PAGE_SIZE;
 378                         allocorder = 0;
 379                         while (bytebufsize > allocsize) {
 380                                 allocsize *= 2;
 381                                 allocorder++;
 382                         }
 383
 384                         s->salami = (void *) devm_get_free_pages(
 385                                 dev,
 386                                 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
 387                                 allocorder);
 388                         if (!s->salami)
 389                                 return -ENOMEM;
 390
 391                         s->left_of_salami = allocsize;
 392                 }
 393
 394                 rc = xilly_map_single(ep, s->salami,
 395                                       bytebufsize, s->direction,
 396                                       &dma_addr);
 397                 if (rc)
 398                         return rc;
 399
 400                 iowrite32((u32) (dma_addr & 0xffffffff),
 401                           ep->registers + fpga_dma_bufaddr_lowaddr_reg);
 402                 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
 403                           ep->registers + fpga_dma_bufaddr_highaddr_reg);
 404
 405                 if (buffers) { /* Not the message buffer */
 406                         this_buffer->addr = s->salami;
 407                         this_buffer->dma_addr = dma_addr;
 408                         buffers[i] = this_buffer++;
 409
 410                         iowrite32(s->regdirection | s->nbuffer++,
 411                                   ep->registers + fpga_dma_bufno_reg);
 412                 } else {
 413                         ep->msgbuf_addr = s->salami;
 414                         ep->msgbuf_dma_addr = dma_addr;
 415                         ep->msg_buf_size = bytebufsize;
 416
 417                         iowrite32(s->regdirection,
 418                                   ep->registers + fpga_dma_bufno_reg);
 419                 }
 420
 421                 s->left_of_salami -= bytebufsize;
 422                 s->salami += bytebufsize;
 423         }
 424         return 0;
 425 }
 426
 427 static int xilly_setupchannels(struct xilly_endpoint *ep,
 428                                unsigned char *chandesc,
 429                                int entries)
 430 {
 431         struct device *dev = ep->dev;
 432         int i, entry, rc;
 433         struct xilly_channel *channel;
 434         int channelnum, bufnum, bufsize, format, is_writebuf;
 435         int bytebufsize;
 436         int synchronous, allowpartial, exclusive_open, seekable;
 437         int supports_nonempty;
 438         int msg_buf_done = 0;
 439
 440         struct xilly_alloc_state rd_alloc = {
 441                 .salami = NULL,
 442                 .left_of_salami = 0,
 443                 .nbuffer = 1,
 444                 .direction = DMA_TO_DEVICE,
 445                 .regdirection = 0,
 446         };
 447
 448         struct xilly_alloc_state wr_alloc = {
 449                 .salami = NULL,
 450                 .left_of_salami = 0,
 451                 .nbuffer = 1,
 452                 .direction = DMA_FROM_DEVICE,
 453                 .regdirection = 0x80000000,
 454         };
 455
 456         channel = devm_kcalloc(dev, ep->num_channels,
 457                                sizeof(struct xilly_channel), GFP_KERNEL);
 458         if (!channel)
 459                 return -ENOMEM;
 460
 461         ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
 462                                     sizeof(struct xilly_channel *),
 463                                     GFP_KERNEL);
 464         if (!ep->channels)
 465                 return -ENOMEM;
 466
 467         ep->channels[0] = NULL; /* Channel 0 is message buf. */
 468
 469         /* Initialize all channels with defaults */
 470
 471         for (i = 1; i <= ep->num_channels; i++) {
 472                 channel->wr_buffers = NULL;
 473                 channel->rd_buffers = NULL;
 474                 channel->num_wr_buffers = 0;
 475                 channel->num_rd_buffers = 0;
 476                 channel->wr_fpga_buf_idx = -1;
 477                 channel->wr_host_buf_idx = 0;
 478                 channel->wr_host_buf_pos = 0;
 479                 channel->wr_empty = 1;
 480                 channel->wr_ready = 0;
 481                 channel->wr_sleepy = 1;
 482                 channel->rd_fpga_buf_idx = 0;
 483                 channel->rd_host_buf_idx = 0;
 484                 channel->rd_host_buf_pos = 0;
 485                 channel->rd_full = 0;
 486                 channel->wr_ref_count = 0;
 487                 channel->rd_ref_count = 0;
 488
 489                 spin_lock_init(&channel->wr_spinlock);
 490                 spin_lock_init(&channel->rd_spinlock);
 491                 mutex_init(&channel->wr_mutex);
 492                 mutex_init(&channel->rd_mutex);
 493                 init_waitqueue_head(&channel->rd_wait);
 494                 init_waitqueue_head(&channel->wr_wait);
 495                 init_waitqueue_head(&channel->wr_ready_wait);
 496
 497                 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 498
 499                 channel->endpoint = ep;
 500                 channel->chan_num = i;
 501
 502                 channel->log2_element_size = 0;
 503
 504                 ep->channels[i] = channel++;
 505         }
 506
 507         for (entry = 0; entry < entries; entry++, chandesc += 4) {
 508                 struct xilly_buffer **buffers = NULL;
 509
 510                 is_writebuf = chandesc[0] & 0x01;
 511                 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 512                 format = (chandesc[1] >> 4) & 0x03;
 513                 allowpartial = (chandesc[1] >> 6) & 0x01;
 514                 synchronous = (chandesc[1] >> 7) & 0x01;
 515                 bufsize = 1 << (chandesc[2] & 0x1f);
 516                 bufnum = 1 << (chandesc[3] & 0x0f);
 517                 exclusive_open = (chandesc[2] >> 7) & 0x01;
 518                 seekable = (chandesc[2] >> 6) & 0x01;
 519                 supports_nonempty = (chandesc[2] >> 5) & 0x01;
 520
 521                 if ((channelnum > ep->num_channels) ||
 522                     ((channelnum == 0) && !is_writebuf)) {
 523                         dev_err(ep->dev,
 524                                 "IDT requests channel out of range. Aborting.\n");
 525                         return -ENODEV;
 526                 }
 527
 528                 channel = ep->channels[channelnum]; /* NULL for msg channel */
 529
 530                 if (!is_writebuf || channelnum > 0) {
 531                         channel->log2_element_size = ((format > 2) ?
 532                                                       2 : format);
 533
 534                         bytebufsize = bufsize *
 535                                 (1 << channel->log2_element_size);
 536
 537                         buffers = devm_kcalloc(dev, bufnum,
 538                                                sizeof(struct xilly_buffer *),
 539                                                GFP_KERNEL);
 540                         if (!buffers)
 541                                 return -ENOMEM;
 542                 } else {
 543                         bytebufsize = bufsize << 2;
 544                 }
 545
 546                 if (!is_writebuf) {
 547                         channel->num_rd_buffers = bufnum;
 548                         channel->rd_buf_size = bytebufsize;
 549                         channel->rd_allow_partial = allowpartial;
 550                         channel->rd_synchronous = synchronous;
 551                         channel->rd_exclusive_open = exclusive_open;
 552                         channel->seekable = seekable;
 553
 554                         channel->rd_buffers = buffers;
 555                         rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
 556                                                    bufnum, bytebufsize);
 557                 } else if (channelnum > 0) {
 558                         channel->num_wr_buffers = bufnum;
 559                         channel->wr_buf_size = bytebufsize;
 560
 561                         channel->seekable = seekable;
 562                         channel->wr_supports_nonempty = supports_nonempty;
 563
 564                         channel->wr_allow_partial = allowpartial;
 565                         channel->wr_synchronous = synchronous;
 566                         channel->wr_exclusive_open = exclusive_open;
 567
 568                         channel->wr_buffers = buffers;
 569                         rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
 570                                                    bufnum, bytebufsize);
 571                 } else {
 572                         rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
 573                                                    bufnum, bytebufsize);
 574                         msg_buf_done++;
 575                 }
 576
 577                 if (rc)
 578                         return -ENOMEM;
 579         }
 580
 581         if (!msg_buf_done) {
 582                 dev_err(ep->dev,
 583                         "Corrupt IDT: No message buffer. Aborting.\n");
 584                 return -ENODEV;
 585         }
 586         return 0;
 587 }
 588
 589 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
 590                           struct xilly_idt_handle *idt_handle)
 591 {
 592         int count = 0;
 593         unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 594         unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 595         unsigned char *scan;
 596         int len;
 597
 598         scan = idt + 1;
 599         idt_handle->names = scan;
 600
 601         while ((scan <= end_of_idt) && *scan) {
 602                 while ((scan <= end_of_idt) && *scan++)
 603                         /* Do nothing, just scan thru string */;
 604                 count++;
 605         }
 606
 607         idt_handle->names_len = scan - idt_handle->names;
 608
 609         scan++;
 610
 611         if (scan > end_of_idt) {
 612                 dev_err(endpoint->dev,
 613                         "IDT device name list overflow. Aborting.\n");
 614                 return -ENODEV;
 615         }
 616         idt_handle->chandesc = scan;
 617
 618         len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 619
 620         if (len & 0x03) {
 621                 dev_err(endpoint->dev,
 622                         "Corrupt IDT device name list. Aborting.\n");
 623                 return -ENODEV;
 624         }
 625
 626         idt_handle->entries = len >> 2;
 627         endpoint->num_channels = count;
 628
 629         return 0;
 630 }
 631
 632 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 633 {
 634         struct xilly_channel *channel;
 635         unsigned char *version;
 636         long t;
 637
 638         channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 639
 640         channel->wr_sleepy = 1;
 641
 642         iowrite32(1 |
 643                   (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 644                   endpoint->registers + fpga_buf_ctrl_reg);
 645
 646         t = wait_event_interruptible_timeout(channel->wr_wait,
 647                                              (!channel->wr_sleepy),
 648                                              XILLY_TIMEOUT);
 649
 650         if (t <= 0) {
 651                 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 652
 653                 if (endpoint->fatal_error)
 654                         return -EIO;
 655
 656                 return -ENODEV;
 657         }
 658
 659         dma_sync_single_for_cpu(channel->endpoint->dev,
 660                                 channel->wr_buffers[0]->dma_addr,
 661                                 channel->wr_buf_size,
 662                                 DMA_FROM_DEVICE);
 663
 664         if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 665                 dev_err(endpoint->dev,
 666                         "IDT length mismatch (%d != %d). Aborting.\n",
 667                         channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 668                 return -ENODEV;
 669         }
 670
 671         if (crc32_le(~0, channel->wr_buffers[0]->addr,
 672                      endpoint->idtlen+1) != 0) {
 673                 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 674                 return -ENODEV;
 675         }
 676
 677         version = channel->wr_buffers[0]->addr;
 678
 679         /* Check version number. Reject anything above 0x82. */
 680         if (*version > 0x82) {
 681                 dev_err(endpoint->dev,
 682                         "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
 683                         *version);
 684                 return -ENODEV;
 685         }
 686
 687         return 0;
 688 }
 689
 690 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 691                              size_t count, loff_t *f_pos)
 692 {
 693         ssize_t rc;
 694         unsigned long flags;
 695         int bytes_done = 0;
 696         int no_time_left = 0;
 697         long deadline, left_to_sleep;
 698         struct xilly_channel *channel = filp->private_data;
 699
 700         int empty, reached_eof, exhausted, ready;
 701         /* Initializations are there only to silence warnings */
 702
 703         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 704         int waiting_bufidx;
 705
 706         if (channel->endpoint->fatal_error)
 707                 return -EIO;
 708
 709         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 710
 711         rc = mutex_lock_interruptible(&channel->wr_mutex);
 712         if (rc)
 713                 return rc;
 714
 715         while (1) { /* Note that we may drop mutex within this loop */
 716                 int bytes_to_do = count - bytes_done;
 717
 718                 spin_lock_irqsave(&channel->wr_spinlock, flags);
 719
 720                 empty = channel->wr_empty;
 721                 ready = !empty || channel->wr_ready;
 722
 723                 if (!empty) {
 724                         bufidx = channel->wr_host_buf_idx;
 725                         bufpos = channel->wr_host_buf_pos;
 726                         howmany = ((channel->wr_buffers[bufidx]->end_offset
 727                                     + 1) << channel->log2_element_size)
 728                                 - bufpos;
 729
 730                         /* Update wr_host_* to its post-operation state */
 731                         if (howmany > bytes_to_do) {
 732                                 bufferdone = 0;
 733
 734                                 howmany = bytes_to_do;
 735                                 channel->wr_host_buf_pos += howmany;
 736                         } else {
 737                                 bufferdone = 1;
 738
 739                                 channel->wr_host_buf_pos = 0;
 740
 741                                 if (bufidx == channel->wr_fpga_buf_idx) {
 742                                         channel->wr_empty = 1;
 743                                         channel->wr_sleepy = 1;
 744                                         channel->wr_ready = 0;
 745                                 }
 746
 747                                 if (bufidx >= (channel->num_wr_buffers - 1))
 748                                         channel->wr_host_buf_idx = 0;
 749                                 else
 750                                         channel->wr_host_buf_idx++;
 751                         }
 752                 }
 753
 754                 /*
 755                  * Marking our situation after the possible changes above,
 756                  * for use after releasing the spinlock.
 757                  *
 758                  * empty = empty before change
 759                  * exhasted = empty after possible change
 760                  */
 761
 762                 reached_eof = channel->wr_empty &&
 763                         (channel->wr_host_buf_idx == channel->wr_eof);
 764                 channel->wr_hangup = reached_eof;
 765                 exhausted = channel->wr_empty;
 766                 waiting_bufidx = channel->wr_host_buf_idx;
 767
 768                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 769
 770                 if (!empty) { /* Go on, now without the spinlock */
 771
 772                         if (bufpos == 0) /* Position zero means it's virgin */
 773                                 dma_sync_single_for_cpu(channel->endpoint->dev,
 774                                                         channel->wr_buffers[bufidx]->dma_addr,
 775                                                         channel->wr_buf_size,
 776                                                         DMA_FROM_DEVICE);
 777
 778                         if (copy_to_user(
 779                                     userbuf,
 780                                     channel->wr_buffers[bufidx]->addr
 781                                     + bufpos, howmany))
 782                                 rc = -EFAULT;
 783
 784                         userbuf += howmany;
 785                         bytes_done += howmany;
 786
 787                         if (bufferdone) {
 788                                 dma_sync_single_for_device(channel->endpoint->dev,
 789                                                            channel->wr_buffers[bufidx]->dma_addr,
 790                                                            channel->wr_buf_size,
 791                                                            DMA_FROM_DEVICE);
 792
 793                                 /*
 794                                  * Tell FPGA the buffer is done with. It's an
 795                                  * atomic operation to the FPGA, so what
 796                                  * happens with other channels doesn't matter,
 797                                  * and the certain channel is protected with
 798                                  * the channel-specific mutex.
 799                                  */
 800
 801                                 iowrite32(1 | (channel->chan_num << 1) |
 802                                           (bufidx << 12),
 803                                           channel->endpoint->registers +
 804                                           fpga_buf_ctrl_reg);
 805                         }
 806
 807                         if (rc) {
 808                                 mutex_unlock(&channel->wr_mutex);
 809                                 return rc;
 810                         }
 811                 }
 812
 813                 /* This includes a zero-count return = EOF */
 814                 if ((bytes_done >= count) || reached_eof)
 815                         break;
 816
 817                 if (!exhausted)
 818                         continue; /* More in RAM buffer(s)? Just go on. */
 819
 820                 if ((bytes_done > 0) &&
 821                     (no_time_left ||
 822                      (channel->wr_synchronous && channel->wr_allow_partial)))
 823                         break;
 824
 825                 /*
 826                  * Nonblocking read: The "ready" flag tells us that the FPGA
 827                  * has data to send. In non-blocking mode, if it isn't on,
 828                  * just return. But if there is, we jump directly to the point
 829                  * where we ask for the FPGA to send all it has, and wait
 830                  * until that data arrives. So in a sense, we *do* block in
 831                  * nonblocking mode, but only for a very short time.
 832                  */
 833
 834                 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 835                         if (bytes_done > 0)
 836                                 break;
 837
 838                         if (ready)
 839                                 goto desperate;
 840
 841                         rc = -EAGAIN;
 842                         break;
 843                 }
 844
 845                 if (!no_time_left || (bytes_done > 0)) {
 846                         /*
 847                          * Note that in case of an element-misaligned read
 848                          * request, offsetlimit will include the last element,
 849                          * which will be partially read from.
 850                          */
 851                         int offsetlimit = ((count - bytes_done) - 1) >>
 852                                 channel->log2_element_size;
 853                         int buf_elements = channel->wr_buf_size >>
 854                                 channel->log2_element_size;
 855
 856                         /*
 857                          * In synchronous mode, always send an offset limit.
 858                          * Just don't send a value too big.
 859                          */
 860
 861                         if (channel->wr_synchronous) {
 862                                 /* Don't request more than one buffer */
 863                                 if (channel->wr_allow_partial &&
 864                                     (offsetlimit >= buf_elements))
 865                                         offsetlimit = buf_elements - 1;
 866
 867                                 /* Don't request more than all buffers */
 868                                 if (!channel->wr_allow_partial &&
 869                                     (offsetlimit >=
 870                                      (buf_elements * channel->num_wr_buffers)))
 871                                         offsetlimit = buf_elements *
 872                                                 channel->num_wr_buffers - 1;
 873                         }
 874
 875                         /*
 876                          * In asynchronous mode, force early flush of a buffer
 877                          * only if that will allow returning a full count. The
 878                          * "offsetlimit < ( ... )" rather than "<=" excludes
 879                          * requesting a full buffer, which would obviously
 880                          * cause a buffer transmission anyhow
 881                          */
 882
 883                         if (channel->wr_synchronous ||
 884                             (offsetlimit < (buf_elements - 1))) {
 885                                 mutex_lock(&channel->endpoint->register_mutex);
 886
 887                                 iowrite32(offsetlimit,
 888                                           channel->endpoint->registers +
 889                                           fpga_buf_offset_reg);
 890
 891                                 iowrite32(1 | (channel->chan_num << 1) |
 892                                           (2 << 24) |  /* 2 = offset limit */
 893                                           (waiting_bufidx << 12),
 894                                           channel->endpoint->registers +
 895                                           fpga_buf_ctrl_reg);
 896
 897                                 mutex_unlock(&channel->endpoint->
 898                                              register_mutex);
 899                         }
 900                 }
 901
 902                 /*
 903                  * If partial completion is disallowed, there is no point in
 904                  * timeout sleeping. Neither if no_time_left is set and
 905                  * there's no data.
 906                  */
 907
 908                 if (!channel->wr_allow_partial ||
 909                     (no_time_left && (bytes_done == 0))) {
 910                         /*
 911                          * This do-loop will run more than once if another
 912                          * thread reasserted wr_sleepy before we got the mutex
 913                          * back, so we try again.
 914                          */
 915
 916                         do {
 917                                 mutex_unlock(&channel->wr_mutex);
 918
 919                                 if (wait_event_interruptible(
 920                                             channel->wr_wait,
 921                                             (!channel->wr_sleepy)))
 922                                         goto interrupted;
 923
 924                                 if (mutex_lock_interruptible(
 925                                             &channel->wr_mutex))
 926                                         goto interrupted;
 927                         } while (channel->wr_sleepy);
 928
 929                         continue;
 930
 931 interrupted: /* Mutex is not held if got here */
 932                         if (channel->endpoint->fatal_error)
 933                                 return -EIO;
 934                         if (bytes_done)
 935                                 return bytes_done;
 936                         if (filp->f_flags & O_NONBLOCK)
 937                                 return -EAGAIN; /* Don't admit snoozing */
 938                         return -EINTR;
 939                 }
 940
 941                 left_to_sleep = deadline - ((long) jiffies);
 942
 943                 /*
 944                  * If our time is out, skip the waiting. We may miss wr_sleepy
 945                  * being deasserted but hey, almost missing the train is like
 946                  * missing it.
 947                  */
 948
 949                 if (left_to_sleep > 0) {
 950                         left_to_sleep =
 951                                 wait_event_interruptible_timeout(
 952                                         channel->wr_wait,
 953                                         (!channel->wr_sleepy),
 954                                         left_to_sleep);
 955
 956                         if (left_to_sleep > 0) /* wr_sleepy deasserted */
 957                                 continue;
 958
 959                         if (left_to_sleep < 0) { /* Interrupt */
 960                                 mutex_unlock(&channel->wr_mutex);
 961                                 if (channel->endpoint->fatal_error)
 962                                         return -EIO;
 963                                 if (bytes_done)
 964                                         return bytes_done;
 965                                 return -EINTR;
 966                         }
 967                 }
 968
 969 desperate:
 970                 no_time_left = 1; /* We're out of sleeping time. Desperate! */
 971
 972                 if (bytes_done == 0) {
 973                         /*
 974                          * Reaching here means that we allow partial return,
 975                          * that we've run out of time, and that we have
 976                          * nothing to return.
 977                          * So tell the FPGA to send anything it has or gets.
 978                          */
 979
 980                         iowrite32(1 | (channel->chan_num << 1) |
 981                                   (3 << 24) |  /* Opcode 3, flush it all! */
 982                                   (waiting_bufidx << 12),
 983                                   channel->endpoint->registers +
 984                                   fpga_buf_ctrl_reg);
 985                 }
 986
 987                 /*
 988                  * Reaching here means that we *do* have data in the buffer,
 989                  * but the "partial" flag disallows returning less than
 990                  * required. And we don't have as much. So loop again,
 991                  * which is likely to end up blocking indefinitely until
 992                  * enough data has arrived.
 993                  */
 994         }
 995
 996         mutex_unlock(&channel->wr_mutex);
 997
 998         if (channel->endpoint->fatal_error)
 999                 return -EIO;
1000
1001         if (rc)
1002                 return rc;
1003
1004         return bytes_done;
1005 }
1006
1007 /*
1008  * The timeout argument takes values as follows:
1009  *  >0 : Flush with timeout
1010  * ==0 : Flush, and wait idefinitely for the flush to complete
1011  *  <0 : Autoflush: Flush only if there's a single buffer occupied
1012  */
1013
1014 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
1015 {
1016         int rc;
1017         unsigned long flags;
1018
1019         int end_offset_plus1;
1020         int bufidx, bufidx_minus1;
1021         int i;
1022         int empty;
1023         int new_rd_host_buf_pos;
1024
1025         if (channel->endpoint->fatal_error)
1026                 return -EIO;
1027         rc = mutex_lock_interruptible(&channel->rd_mutex);
1028         if (rc)
1029                 return rc;
1030
1031         /*
1032          * Don't flush a closed channel. This can happen when the work queued
1033          * autoflush thread fires off after the file has closed. This is not
1034          * an error, just something to dismiss.
1035          */
1036
1037         if (!channel->rd_ref_count)
1038                 goto done;
1039
1040         bufidx = channel->rd_host_buf_idx;
1041
1042         bufidx_minus1 = (bufidx == 0) ?
1043                 channel->num_rd_buffers - 1 :
1044                 bufidx - 1;
1045
1046         end_offset_plus1 = channel->rd_host_buf_pos >>
1047                 channel->log2_element_size;
1048
1049         new_rd_host_buf_pos = channel->rd_host_buf_pos -
1050                 (end_offset_plus1 << channel->log2_element_size);
1051
1052         /* Submit the current buffer if it's nonempty */
1053         if (end_offset_plus1) {
1054                 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1055                         (end_offset_plus1 << channel->log2_element_size);
1056
1057                 /* Copy  unflushed data, so we can put it in next buffer */
1058                 for (i = 0; i < new_rd_host_buf_pos; i++)
1059                         channel->rd_leftovers[i] = *tail++;
1060
1061                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1062
1063                 /* Autoflush only if a single buffer is occupied */
1064
1065                 if ((timeout < 0) &&
1066                     (channel->rd_full ||
1067                      (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1068                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1069                         /*
1070                          * A new work item may be queued by the ISR exactly
1071                          * now, since the execution of a work item allows the
1072                          * queuing of a new one while it's running.
1073                          */
1074                         goto done;
1075                 }
1076
1077                 /* The 4th element is never needed for data, so it's a flag */
1078                 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1079
1080                 /* Set up rd_full to reflect a certain moment's state */
1081
1082                 if (bufidx == channel->rd_fpga_buf_idx)
1083                         channel->rd_full = 1;
1084                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1085
1086                 if (bufidx >= (channel->num_rd_buffers - 1))
1087                         channel->rd_host_buf_idx = 0;
1088                 else
1089                         channel->rd_host_buf_idx++;
1090
1091                 dma_sync_single_for_device(channel->endpoint->dev,
1092                                            channel->rd_buffers[bufidx]->dma_addr,
1093                                            channel->rd_buf_size,
1094                                            DMA_TO_DEVICE);
1095
1096                 mutex_lock(&channel->endpoint->register_mutex);
1097
1098                 iowrite32(end_offset_plus1 - 1,
1099                           channel->endpoint->registers + fpga_buf_offset_reg);
1100
1101                 iowrite32((channel->chan_num << 1) | /* Channel ID */
1102                           (2 << 24) |  /* Opcode 2, submit buffer */
1103                           (bufidx << 12),
1104                           channel->endpoint->registers + fpga_buf_ctrl_reg);
1105
1106                 mutex_unlock(&channel->endpoint->register_mutex);
1107         } else if (bufidx == 0) {
1108                 bufidx = channel->num_rd_buffers - 1;
1109         } else {
1110                 bufidx--;
1111         }
1112
1113         channel->rd_host_buf_pos = new_rd_host_buf_pos;
1114
1115         if (timeout < 0)
1116                 goto done; /* Autoflush */
1117
1118         /*
1119          * bufidx is now the last buffer written to (or equal to
1120          * rd_fpga_buf_idx if buffer was never written to), and
1121          * channel->rd_host_buf_idx the one after it.
1122          *
1123          * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1124          */
1125
1126         while (1) { /* Loop waiting for draining of buffers */
1127                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1128
1129                 if (bufidx != channel->rd_fpga_buf_idx)
1130                         channel->rd_full = 1; /*
1131                                                * Not really full,
1132                                                * but needs waiting.
1133                                                */
1134
1135                 empty = !channel->rd_full;
1136
1137                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1138
1139                 if (empty)
1140                         break;
1141
1142                 /*
1143                  * Indefinite sleep with mutex taken. With data waiting for
1144                  * flushing user should not be surprised if open() for write
1145                  * sleeps.
1146                  */
1147                 if (timeout == 0)
1148                         wait_event_interruptible(channel->rd_wait,
1149                                                  (!channel->rd_full));
1150
1151                 else if (wait_event_interruptible_timeout(
1152                                  channel->rd_wait,
1153                                  (!channel->rd_full),
1154                                  timeout) == 0) {
1155                         dev_warn(channel->endpoint->dev,
1156                                  "Timed out while flushing. Output data may be lost.\n");
1157
1158                         rc = -ETIMEDOUT;
1159                         break;
1160                 }
1161
1162                 if (channel->rd_full) {
1163                         rc = -EINTR;
1164                         break;
1165                 }
1166         }
1167
1168 done:
1169         mutex_unlock(&channel->rd_mutex);
1170
1171         if (channel->endpoint->fatal_error)
1172                 return -EIO;
1173
1174         return rc;
1175 }
1176
1177 static int xillybus_flush(struct file *filp, fl_owner_t id)
1178 {
1179         if (!(filp->f_mode & FMODE_WRITE))
1180                 return 0;
1181
1182         return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1183 }
1184
1185 static void xillybus_autoflush(struct work_struct *work)
1186 {
1187         struct delayed_work *workitem = container_of(
1188                 work, struct delayed_work, work);
1189         struct xilly_channel *channel = container_of(
1190                 workitem, struct xilly_channel, rd_workitem);
1191         int rc;
1192
1193         rc = xillybus_myflush(channel, -1);
1194         if (rc == -EINTR)
1195                 dev_warn(channel->endpoint->dev,
1196                          "Autoflush failed because work queue thread got a signal.\n");
1197         else if (rc)
1198                 dev_err(channel->endpoint->dev,
1199                         "Autoflush failed under weird circumstances.\n");
1200 }
1201
1202 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1203                               size_t count, loff_t *f_pos)
1204 {
1205         ssize_t rc;
1206         unsigned long flags;
1207         int bytes_done = 0;
1208         struct xilly_channel *channel = filp->private_data;
1209
1210         int full, exhausted;
1211         /* Initializations are there only to silence warnings */
1212
1213         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1214         int end_offset_plus1 = 0;
1215
1216         if (channel->endpoint->fatal_error)
1217                 return -EIO;
1218
1219         rc = mutex_lock_interruptible(&channel->rd_mutex);
1220         if (rc)
1221                 return rc;
1222
1223         while (1) {
1224                 int bytes_to_do = count - bytes_done;
1225
1226                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1227
1228                 full = channel->rd_full;
1229
1230                 if (!full) {
1231                         bufidx = channel->rd_host_buf_idx;
1232                         bufpos = channel->rd_host_buf_pos;
1233                         howmany = channel->rd_buf_size - bufpos;
1234
1235                         /*
1236                          * Update rd_host_* to its state after this operation.
1237                          * count=0 means committing the buffer immediately,
1238                          * which is like flushing, but not necessarily block.
1239                          */
1240
1241                         if ((howmany > bytes_to_do) &&
1242                             (count ||
1243                              ((bufpos >> channel->log2_element_size) == 0))) {
1244                                 bufferdone = 0;
1245
1246                                 howmany = bytes_to_do;
1247                                 channel->rd_host_buf_pos += howmany;
1248                         } else {
1249                                 bufferdone = 1;
1250
1251                                 if (count) {
1252                                         end_offset_plus1 =
1253                                                 channel->rd_buf_size >>
1254                                                 channel->log2_element_size;
1255                                         channel->rd_host_buf_pos = 0;
1256                                 } else {
1257                                         unsigned char *tail;
1258                                         int i;
1259
1260                                         howmany = 0;
1261
1262                                         end_offset_plus1 = bufpos >>
1263                                                 channel->log2_element_size;
1264
1265                                         channel->rd_host_buf_pos -=
1266                                                 end_offset_plus1 <<
1267                                                 channel->log2_element_size;
1268
1269                                         tail = channel->
1270                                                 rd_buffers[bufidx]->addr +
1271                                                 (end_offset_plus1 <<
1272                                                  channel->log2_element_size);
1273
1274                                         for (i = 0;
1275                                              i < channel->rd_host_buf_pos;
1276                                              i++)
1277                                                 channel->rd_leftovers[i] =
1278                                                         *tail++;
1279                                 }
1280
1281                                 if (bufidx == channel->rd_fpga_buf_idx)
1282                                         channel->rd_full = 1;
1283
1284                                 if (bufidx >= (channel->num_rd_buffers - 1))
1285                                         channel->rd_host_buf_idx = 0;
1286                                 else
1287                                         channel->rd_host_buf_idx++;
1288                         }
1289                 }
1290
1291                 /*
1292                  * Marking our situation after the possible changes above,
1293                  * for use  after releasing the spinlock.
1294                  *
1295                  * full = full before change
1296                  * exhasted = full after possible change
1297                  */
1298
1299                 exhausted = channel->rd_full;
1300
1301                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1302
1303                 if (!full) { /* Go on, now without the spinlock */
1304                         unsigned char *head =
1305                                 channel->rd_buffers[bufidx]->addr;
1306                         int i;
1307
1308                         if ((bufpos == 0) || /* Zero means it's virgin */
1309                             (channel->rd_leftovers[3] != 0)) {
1310                                 dma_sync_single_for_cpu(channel->endpoint->dev,
1311                                                         channel->rd_buffers[bufidx]->dma_addr,
1312                                                         channel->rd_buf_size,
1313                                                         DMA_TO_DEVICE);
1314
1315                                 /* Virgin, but leftovers are due */
1316                                 for (i = 0; i < bufpos; i++)
1317                                         *head++ = channel->rd_leftovers[i];
1318
1319                                 channel->rd_leftovers[3] = 0; /* Clear flag */
1320                         }
1321
1322                         if (copy_from_user(
1323                                     channel->rd_buffers[bufidx]->addr + bufpos,
1324                                     userbuf, howmany))
1325                                 rc = -EFAULT;
1326
1327                         userbuf += howmany;
1328                         bytes_done += howmany;
1329
1330                         if (bufferdone) {
1331                                 dma_sync_single_for_device(channel->endpoint->dev,
1332                                                            channel->rd_buffers[bufidx]->dma_addr,
1333                                                            channel->rd_buf_size,
1334                                                            DMA_TO_DEVICE);
1335
1336                                 mutex_lock(&channel->endpoint->register_mutex);
1337
1338                                 iowrite32(end_offset_plus1 - 1,
1339                                           channel->endpoint->registers +
1340                                           fpga_buf_offset_reg);
1341
1342                                 iowrite32((channel->chan_num << 1) |
1343                                           (2 << 24) |  /* 2 = submit buffer */
1344                                           (bufidx << 12),
1345                                           channel->endpoint->registers +
1346                                           fpga_buf_ctrl_reg);
1347
1348                                 mutex_unlock(&channel->endpoint->
1349                                              register_mutex);
1350
1351                                 channel->rd_leftovers[3] =
1352                                         (channel->rd_host_buf_pos != 0);
1353                         }
1354
1355                         if (rc) {
1356                                 mutex_unlock(&channel->rd_mutex);
1357
1358                                 if (channel->endpoint->fatal_error)
1359                                         return -EIO;
1360
1361                                 if (!channel->rd_synchronous)
1362                                         queue_delayed_work(
1363                                                 xillybus_wq,
1364                                                 &channel->rd_workitem,
1365                                                 XILLY_RX_TIMEOUT);
1366
1367                                 return rc;
1368                         }
1369                 }
1370
1371                 if (bytes_done >= count)
1372                         break;
1373
1374                 if (!exhausted)
1375                         continue; /* If there's more space, just go on */
1376
1377                 if ((bytes_done > 0) && channel->rd_allow_partial)
1378                         break;
1379
1380                 /*
1381                  * Indefinite sleep with mutex taken. With data waiting for
1382                  * flushing, user should not be surprised if open() for write
1383                  * sleeps.
1384                  */
1385
1386                 if (filp->f_flags & O_NONBLOCK) {
1387                         rc = -EAGAIN;
1388                         break;
1389                 }
1390
1391                 if (wait_event_interruptible(channel->rd_wait,
1392                                              (!channel->rd_full))) {
1393                         mutex_unlock(&channel->rd_mutex);
1394
1395                         if (channel->endpoint->fatal_error)
1396                                 return -EIO;
1397
1398                         if (bytes_done)
1399                                 return bytes_done;
1400                         return -EINTR;
1401                 }
1402         }
1403
1404         mutex_unlock(&channel->rd_mutex);
1405
1406         if (!channel->rd_synchronous)
1407                 queue_delayed_work(xillybus_wq,
1408                                    &channel->rd_workitem,
1409                                    XILLY_RX_TIMEOUT);
1410
1411         if (channel->endpoint->fatal_error)
1412                 return -EIO;
1413
1414         if (rc)
1415                 return rc;
1416
1417         if ((channel->rd_synchronous) && (bytes_done > 0)) {
1418                 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1419
1420                 if (rc && (rc != -EINTR))
1421                         return rc;
1422         }
1423
1424         return bytes_done;
1425 }
1426
1427 static int xillybus_open(struct inode *inode, struct file *filp)
1428 {
1429         int rc;
1430         unsigned long flags;
1431         struct xilly_endpoint *endpoint;
1432         struct xilly_channel *channel;
1433         int index;
1434
1435         rc = xillybus_find_inode(inode, (void **)&endpoint, &index);
1436         if (rc)
1437                 return rc;
1438
1439         if (endpoint->fatal_error)
1440                 return -EIO;
1441
1442         channel = endpoint->channels[1 + index];
1443         filp->private_data = channel;
1444
1445         /*
1446          * It gets complicated because:
1447          * 1. We don't want to take a mutex we don't have to
1448          * 2. We don't want to open one direction if the other will fail.
1449          */
1450
1451         if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1452                 return -ENODEV;
1453
1454         if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1455                 return -ENODEV;
1456
1457         if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1458             (channel->wr_synchronous || !channel->wr_allow_partial ||
1459              !channel->wr_supports_nonempty)) {
1460                 dev_err(endpoint->dev,
1461                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1462                 return -ENODEV;
1463         }
1464
1465         if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1466             (channel->rd_synchronous || !channel->rd_allow_partial)) {
1467                 dev_err(endpoint->dev,
1468                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1469                 return -ENODEV;
1470         }
1471
1472         /*
1473          * Note: open() may block on getting mutexes despite O_NONBLOCK.
1474          * This shouldn't occur normally, since multiple open of the same
1475          * file descriptor is almost always prohibited anyhow
1476          * (*_exclusive_open is normally set in real-life systems).
1477          */
1478
1479         if (filp->f_mode & FMODE_READ) {
1480                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1481                 if (rc)
1482                         return rc;
1483         }
1484
1485         if (filp->f_mode & FMODE_WRITE) {
1486                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1487                 if (rc)
1488                         goto unlock_wr;
1489         }
1490
1491         if ((filp->f_mode & FMODE_READ) &&
1492             (channel->wr_ref_count != 0) &&
1493             (channel->wr_exclusive_open)) {
1494                 rc = -EBUSY;
1495                 goto unlock;
1496         }
1497
1498         if ((filp->f_mode & FMODE_WRITE) &&
1499             (channel->rd_ref_count != 0) &&
1500             (channel->rd_exclusive_open)) {
1501                 rc = -EBUSY;
1502                 goto unlock;
1503         }
1504
1505         if (filp->f_mode & FMODE_READ) {
1506                 if (channel->wr_ref_count == 0) { /* First open of file */
1507                         /* Move the host to first buffer */
1508                         spin_lock_irqsave(&channel->wr_spinlock, flags);
1509                         channel->wr_host_buf_idx = 0;
1510                         channel->wr_host_buf_pos = 0;
1511                         channel->wr_fpga_buf_idx = -1;
1512                         channel->wr_empty = 1;
1513                         channel->wr_ready = 0;
1514                         channel->wr_sleepy = 1;
1515                         channel->wr_eof = -1;
1516                         channel->wr_hangup = 0;
1517
1518                         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1519
1520                         iowrite32(1 | (channel->chan_num << 1) |
1521                                   (4 << 24) |  /* Opcode 4, open channel */
1522                                   ((channel->wr_synchronous & 1) << 23),
1523                                   channel->endpoint->registers +
1524                                   fpga_buf_ctrl_reg);
1525                 }
1526
1527                 channel->wr_ref_count++;
1528         }
1529
1530         if (filp->f_mode & FMODE_WRITE) {
1531                 if (channel->rd_ref_count == 0) { /* First open of file */
1532                         /* Move the host to first buffer */
1533                         spin_lock_irqsave(&channel->rd_spinlock, flags);
1534                         channel->rd_host_buf_idx = 0;
1535                         channel->rd_host_buf_pos = 0;
1536                         channel->rd_leftovers[3] = 0; /* No leftovers. */
1537                         channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1538                         channel->rd_full = 0;
1539
1540                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1541
1542                         iowrite32((channel->chan_num << 1) |
1543                                   (4 << 24),   /* Opcode 4, open channel */
1544                                   channel->endpoint->registers +
1545                                   fpga_buf_ctrl_reg);
1546                 }
1547
1548                 channel->rd_ref_count++;
1549         }
1550
1551 unlock:
1552         if (filp->f_mode & FMODE_WRITE)
1553                 mutex_unlock(&channel->rd_mutex);
1554 unlock_wr:
1555         if (filp->f_mode & FMODE_READ)
1556                 mutex_unlock(&channel->wr_mutex);
1557
1558         if (!rc && (!channel->seekable))
1559                 return nonseekable_open(inode, filp);
1560
1561         return rc;
1562 }
1563
1564 static int xillybus_release(struct inode *inode, struct file *filp)
1565 {
1566         unsigned long flags;
1567         struct xilly_channel *channel = filp->private_data;
1568
1569         int buf_idx;
1570         int eof;
1571
1572         if (channel->endpoint->fatal_error)
1573                 return -EIO;
1574
1575         if (filp->f_mode & FMODE_WRITE) {
1576                 mutex_lock(&channel->rd_mutex);
1577
1578                 channel->rd_ref_count--;
1579
1580                 if (channel->rd_ref_count == 0) {
1581                         /*
1582                          * We rely on the kernel calling flush()
1583                          * before we get here.
1584                          */
1585
1586                         iowrite32((channel->chan_num << 1) | /* Channel ID */
1587                                   (5 << 24),  /* Opcode 5, close channel */
1588                                   channel->endpoint->registers +
1589                                   fpga_buf_ctrl_reg);
1590                 }
1591                 mutex_unlock(&channel->rd_mutex);
1592         }
1593
1594         if (filp->f_mode & FMODE_READ) {
1595                 mutex_lock(&channel->wr_mutex);
1596
1597                 channel->wr_ref_count--;
1598
1599                 if (channel->wr_ref_count == 0) {
1600                         iowrite32(1 | (channel->chan_num << 1) |
1601                                   (5 << 24),  /* Opcode 5, close channel */
1602                                   channel->endpoint->registers +
1603                                   fpga_buf_ctrl_reg);
1604
1605                         /*
1606                          * This is crazily cautious: We make sure that not
1607                          * only that we got an EOF (be it because we closed
1608                          * the channel or because of a user's EOF), but verify
1609                          * that it's one beyond the last buffer arrived, so
1610                          * we have no leftover buffers pending before wrapping
1611                          * up (which can only happen in asynchronous channels,
1612                          * BTW)
1613                          */
1614
1615                         while (1) {
1616                                 spin_lock_irqsave(&channel->wr_spinlock,
1617                                                   flags);
1618                                 buf_idx = channel->wr_fpga_buf_idx;
1619                                 eof = channel->wr_eof;
1620                                 channel->wr_sleepy = 1;
1621                                 spin_unlock_irqrestore(&channel->wr_spinlock,
1622                                                        flags);
1623
1624                                 /*
1625                                  * Check if eof points at the buffer after
1626                                  * the last one the FPGA submitted. Note that
1627                                  * no EOF is marked by negative eof.
1628                                  */
1629
1630                                 buf_idx++;
1631                                 if (buf_idx == channel->num_wr_buffers)
1632                                         buf_idx = 0;
1633
1634                                 if (buf_idx == eof)
1635                                         break;
1636
1637                                 /*
1638                                  * Steal extra 100 ms if awaken by interrupt.
1639                                  * This is a simple workaround for an
1640                                  * interrupt pending when entering, which would
1641                                  * otherwise result in declaring the hardware
1642                                  * non-responsive.
1643                                  */
1644
1645                                 if (wait_event_interruptible(
1646                                             channel->wr_wait,
1647                                             (!channel->wr_sleepy)))
1648                                         msleep(100);
1649
1650                                 if (channel->wr_sleepy) {
1651                                         mutex_unlock(&channel->wr_mutex);
1652                                         dev_warn(channel->endpoint->dev,
1653                                                  "Hardware failed to respond to close command, therefore left in messy state.\n");
1654                                         return -EINTR;
1655                                 }
1656                         }
1657                 }
1658
1659                 mutex_unlock(&channel->wr_mutex);
1660         }
1661
1662         return 0;
1663 }
1664
1665 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1666 {
1667         struct xilly_channel *channel = filp->private_data;
1668         loff_t pos = filp->f_pos;
1669         int rc = 0;
1670
1671         /*
1672          * Take both mutexes not allowing interrupts, since it seems like
1673          * common applications don't expect an -EINTR here. Besides, multiple
1674          * access to a single file descriptor on seekable devices is a mess
1675          * anyhow.
1676          */
1677
1678         if (channel->endpoint->fatal_error)
1679                 return -EIO;
1680
1681         mutex_lock(&channel->wr_mutex);
1682         mutex_lock(&channel->rd_mutex);
1683
1684         switch (whence) {
1685         case SEEK_SET:
1686                 pos = offset;
1687                 break;
1688         case SEEK_CUR:
1689                 pos += offset;
1690                 break;
1691         case SEEK_END:
1692                 pos = offset; /* Going to the end => to the beginning */
1693                 break;
1694         default:
1695                 rc = -EINVAL;
1696                 goto end;
1697         }
1698
1699         /* In any case, we must finish on an element boundary */
1700         if (pos & ((1 << channel->log2_element_size) - 1)) {
1701                 rc = -EINVAL;
1702                 goto end;
1703         }
1704
1705         mutex_lock(&channel->endpoint->register_mutex);
1706
1707         iowrite32(pos >> channel->log2_element_size,
1708                   channel->endpoint->registers + fpga_buf_offset_reg);
1709
1710         iowrite32((channel->chan_num << 1) |
1711                   (6 << 24),  /* Opcode 6, set address */
1712                   channel->endpoint->registers + fpga_buf_ctrl_reg);
1713
1714         mutex_unlock(&channel->endpoint->register_mutex);
1715
1716 end:
1717         mutex_unlock(&channel->rd_mutex);
1718         mutex_unlock(&channel->wr_mutex);
1719
1720         if (rc) /* Return error after releasing mutexes */
1721                 return rc;
1722
1723         filp->f_pos = pos;
1724
1725         /*
1726          * Since seekable devices are allowed only when the channel is
1727          * synchronous, we assume that there is no data pending in either
1728          * direction (which holds true as long as no concurrent access on the
1729          * file descriptor takes place).
1730          * The only thing we may need to throw away is leftovers from partial
1731          * write() flush.
1732          */
1733
1734         channel->rd_leftovers[3] = 0;
1735
1736         return pos;
1737 }
1738
1739 static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
1740 {
1741         struct xilly_channel *channel = filp->private_data;
1742         __poll_t mask = 0;
1743         unsigned long flags;
1744
1745         poll_wait(filp, &channel->endpoint->ep_wait, wait);
1746
1747         /*
1748          * poll() won't play ball regarding read() channels which
1749          * aren't asynchronous and support the nonempty message. Allowing
1750          * that will create situations where data has been delivered at
1751          * the FPGA, and users expecting select() to wake up, which it may
1752          * not.
1753          */
1754
1755         if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1756                 poll_wait(filp, &channel->wr_wait, wait);
1757                 poll_wait(filp, &channel->wr_ready_wait, wait);
1758
1759                 spin_lock_irqsave(&channel->wr_spinlock, flags);
1760                 if (!channel->wr_empty || channel->wr_ready)
1761                         mask |= EPOLLIN | EPOLLRDNORM;
1762
1763                 if (channel->wr_hangup)
1764                         /*
1765                          * Not EPOLLHUP, because its behavior is in the
1766                          * mist, and EPOLLIN does what we want: Wake up
1767                          * the read file descriptor so it sees EOF.
1768                          */
1769                         mask |=  EPOLLIN | EPOLLRDNORM;
1770                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1771         }
1772
1773         /*
1774          * If partial data write is disallowed on a write() channel,
1775          * it's pointless to ever signal OK to write, because is could
1776          * block despite some space being available.
1777          */
1778
1779         if (channel->rd_allow_partial) {
1780                 poll_wait(filp, &channel->rd_wait, wait);
1781
1782                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1783                 if (!channel->rd_full)
1784                         mask |= EPOLLOUT | EPOLLWRNORM;
1785                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1786         }
1787
1788         if (channel->endpoint->fatal_error)
1789                 mask |= EPOLLERR;
1790
1791         return mask;
1792 }
1793
1794 static const struct file_operations xillybus_fops = {
1795         .owner      = THIS_MODULE,
1796         .read       = xillybus_read,
1797         .write      = xillybus_write,
1798         .open       = xillybus_open,
1799         .flush      = xillybus_flush,
1800         .release    = xillybus_release,
1801         .llseek     = xillybus_llseek,
1802         .poll       = xillybus_poll,
1803 };
1804
1805 struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)
1806 {
1807         struct xilly_endpoint *endpoint;
1808
1809         endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1810         if (!endpoint)
1811                 return NULL;
1812
1813         endpoint->dev = dev;
1814         endpoint->msg_counter = 0x0b;
1815         endpoint->failed_messages = 0;
1816         endpoint->fatal_error = 0;
1817
1818         init_waitqueue_head(&endpoint->ep_wait);
1819         mutex_init(&endpoint->register_mutex);
1820
1821         return endpoint;
1822 }
1823 EXPORT_SYMBOL(xillybus_init_endpoint);
1824
1825 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1826 {
1827         long t;
1828
1829         endpoint->idtlen = -1;
1830
1831         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1832                   endpoint->registers + fpga_dma_control_reg);
1833
1834         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1835                                              (endpoint->idtlen >= 0),
1836                                              XILLY_TIMEOUT);
1837         if (t <= 0) {
1838                 dev_err(endpoint->dev,
1839                         "Failed to quiesce the device on exit.\n");
1840                 return -ENODEV;
1841         }
1842         return 0;
1843 }
1844
1845 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1846 {
1847         int rc;
1848         long t;
1849
1850         void *bootstrap_resources;
1851         int idtbuffersize = (1 << PAGE_SHIFT);
1852         struct device *dev = endpoint->dev;
1853
1854         /*
1855          * The bogus IDT is used during bootstrap for allocating the initial
1856          * message buffer, and then the message buffer and space for the IDT
1857          * itself. The initial message buffer is of a single page's size, but
1858          * it's soon replaced with a more modest one (and memory is freed).
1859          */
1860
1861         unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1862                                        3, 192, PAGE_SHIFT, 0 };
1863         struct xilly_idt_handle idt_handle;
1864
1865         /*
1866          * Writing the value 0x00000001 to Endianness register signals which
1867          * endianness this processor is using, so the FPGA can swap words as
1868          * necessary.
1869          */
1870
1871         iowrite32(1, endpoint->registers + fpga_endian_reg);
1872
1873         /* Bootstrap phase I: Allocate temporary message buffer */
1874
1875         bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1876         if (!bootstrap_resources)
1877                 return -ENOMEM;
1878
1879         endpoint->num_channels = 0;
1880
1881         rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1882         if (rc)
1883                 return rc;
1884
1885         /* Clear the message subsystem (and counter in particular) */
1886         iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1887
1888         endpoint->idtlen = -1;
1889
1890         /*
1891          * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1892          * buffer size.
1893          */
1894         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1895                   endpoint->registers + fpga_dma_control_reg);
1896
1897         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1898                                              (endpoint->idtlen >= 0),
1899                                              XILLY_TIMEOUT);
1900         if (t <= 0) {
1901                 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1902                 return -ENODEV;
1903         }
1904
1905         /* Enable DMA */
1906         iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1907                   endpoint->registers + fpga_dma_control_reg);
1908
1909         /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
1910         while (endpoint->idtlen >= idtbuffersize) {
1911                 idtbuffersize *= 2;
1912                 bogus_idt[6]++;
1913         }
1914
1915         endpoint->num_channels = 1;
1916
1917         rc = xilly_setupchannels(endpoint, bogus_idt, 2);
1918         if (rc)
1919                 goto failed_idt;
1920
1921         rc = xilly_obtain_idt(endpoint);
1922         if (rc)
1923                 goto failed_idt;
1924
1925         rc = xilly_scan_idt(endpoint, &idt_handle);
1926         if (rc)
1927                 goto failed_idt;
1928
1929         devres_close_group(dev, bootstrap_resources);
1930
1931         /* Bootstrap phase III: Allocate buffers according to IDT */
1932
1933         rc = xilly_setupchannels(endpoint,
1934                                  idt_handle.chandesc,
1935                                  idt_handle.entries);
1936         if (rc)
1937                 goto failed_idt;
1938
1939         rc = xillybus_init_chrdev(dev, &xillybus_fops,
1940                                   endpoint->owner, endpoint,
1941                                   idt_handle.names,
1942                                   idt_handle.names_len,
1943                                   endpoint->num_channels,
1944                                   xillyname, false);
1945
1946         if (rc)
1947                 goto failed_idt;
1948
1949         devres_release_group(dev, bootstrap_resources);
1950
1951         return 0;
1952
1953 failed_idt:
1954         xilly_quiesce(endpoint);
1955         flush_workqueue(xillybus_wq);
1956
1957         return rc;
1958 }
1959 EXPORT_SYMBOL(xillybus_endpoint_discovery);
1960
1961 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
1962 {
1963         xillybus_cleanup_chrdev(endpoint, endpoint->dev);
1964
1965         xilly_quiesce(endpoint);
1966
1967         /*
1968          * Flushing is done upon endpoint release to prevent access to memory
1969          * just about to be released. This makes the quiesce complete.
1970          */
1971         flush_workqueue(xillybus_wq);
1972 }
1973 EXPORT_SYMBOL(xillybus_endpoint_remove);
1974
1975 static int __init xillybus_init(void)
1976 {
1977         xillybus_wq = alloc_workqueue(xillyname, 0, 0);
1978         if (!xillybus_wq)
1979                 return -ENOMEM;
1980
1981         return 0;
1982 }
1983
1984 static void __exit xillybus_exit(void)
1985 {
1986         /* flush_workqueue() was called for each endpoint released */
1987         destroy_workqueue(xillybus_wq);
1988 }
1989
1990 module_init(xillybus_init);
1991 module_exit(xillybus_exit);