1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2020 Xillybus Ltd, http://xillybus.com
5 * Driver for the XillyUSB FPGA/host framework.
7 * This driver interfaces with a special IP core in an FPGA, setting up
8 * a pipe between a hardware FIFO in the programmable logic and a device
9 * file in the host. The number of such pipes and their attributes are
10 * set up on the logic. This driver detects these automatically and
11 * creates the device files accordingly.
14 #include <linux/types.h>
15 #include <linux/slab.h>
16 #include <linux/list.h>
17 #include <linux/device.h>
18 #include <linux/module.h>
19 #include <asm/byteorder.h>
21 #include <linux/interrupt.h>
22 #include <linux/sched.h>
24 #include <linux/spinlock.h>
25 #include <linux/mutex.h>
26 #include <linux/workqueue.h>
27 #include <linux/crc32.h>
28 #include <linux/poll.h>
29 #include <linux/delay.h>
30 #include <linux/usb.h>
32 #include "xillybus_class.h"
34 MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core");
35 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
36 MODULE_ALIAS("xillyusb");
37 MODULE_LICENSE("GPL v2");
39 #define XILLY_RX_TIMEOUT (10 * HZ / 1000)
40 #define XILLY_RESPONSE_TIMEOUT (500 * HZ / 1000)
42 #define BUF_SIZE_ORDER 4
44 #define LOG2_IDT_FIFO_SIZE 16
45 #define LOG2_INITIAL_FIFO_BUF_SIZE 16
50 static const char xillyname[] = "xillyusb";
52 static unsigned int fifo_buf_order;
54 #define USB_VENDOR_ID_XILINX 0x03fd
55 #define USB_VENDOR_ID_ALTERA 0x09fb
57 #define USB_PRODUCT_ID_XILLYUSB 0xebbe
59 static const struct usb_device_id xillyusb_table[] = {
60 { USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) },
61 { USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) },
65 MODULE_DEVICE_TABLE(usb, xillyusb_table);
70 unsigned int bufsize; /* In bytes, always a power of 2 */
72 unsigned int size; /* Lazy: Equals bufsize * bufnum */
73 unsigned int buf_order;
75 int fill; /* Number of bytes in the FIFO */
77 wait_queue_head_t waitq;
81 unsigned int writepos;
82 unsigned int writebuf;
86 struct xillyusb_channel;
88 struct xillyusb_endpoint {
89 struct xillyusb_dev *xdev;
91 struct mutex ep_mutex; /* serialize operations on endpoint */
93 struct list_head buffers;
94 struct list_head filled_buffers;
95 spinlock_t buffers_lock; /* protect these two lists */
98 unsigned int buffer_size;
100 unsigned int fill_mask;
102 int outstanding_urbs;
104 struct usb_anchor anchor;
106 struct xillyfifo fifo;
108 struct work_struct workitem;
117 struct xillyusb_channel {
118 struct xillyusb_dev *xdev;
120 struct xillyfifo *in_fifo;
121 struct xillyusb_endpoint *out_ep;
122 struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */
124 struct mutex in_mutex; /* serialize fops on FPGA to host stream */
125 struct mutex out_mutex; /* serialize fops on host to FPGA stream */
126 wait_queue_head_t flushq;
130 u32 in_consumed_bytes;
131 u32 in_current_checkpoint;
134 unsigned int in_log2_element_size;
135 unsigned int out_log2_element_size;
136 unsigned int in_log2_fifo_size;
137 unsigned int out_log2_fifo_size;
139 unsigned int read_data_ok; /* EOF not arrived (yet) */
140 unsigned int poll_used;
141 unsigned int flushing;
142 unsigned int flushed;
143 unsigned int canceled;
145 /* Bit fields protected by @lock except for initialization */
148 unsigned open_for_read:1;
149 unsigned open_for_write:1;
150 unsigned in_synchronous:1;
151 unsigned out_synchronous:1;
152 unsigned in_seekable:1;
153 unsigned out_seekable:1;
157 struct list_head entry;
158 struct xillyusb_endpoint *ep;
163 struct xillyusb_dev {
164 struct xillyusb_channel *channels;
166 struct usb_device *udev;
167 struct device *dev; /* For dev_err() and such */
169 struct workqueue_struct *workq;
172 spinlock_t error_lock; /* protect @error */
173 struct work_struct wakeup_workitem;
177 struct xillyusb_endpoint *msg_ep;
178 struct xillyusb_endpoint *in_ep;
180 struct mutex msg_mutex; /* serialize opcode transmission */
182 int leftover_chan_num;
183 unsigned int in_counter;
184 struct mutex process_in_mutex; /* synchronize wakeup_all() */
188 * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev
189 * struct from being freed during the gap between being found by
190 * xillybus_find_inode() and having its reference count incremented.
193 static DEFINE_MUTEX(kref_mutex);
195 /* FPGA to host opcodes */
198 OPCODE_QUIESCE_ACK = 1,
200 OPCODE_REACHED_CHECKPOINT = 3,
201 OPCODE_CANCELED_CHECKPOINT = 4,
204 /* Host to FPGA opcodes */
208 OPCODE_SET_CHECKPOINT = 2,
211 OPCODE_UPDATE_PUSH = 5,
212 OPCODE_CANCEL_CHECKPOINT = 6,
217 * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple
218 * calls to each on the same FIFO is not allowed) however it's OK to have
219 * threads calling each of the two functions once on the same FIFO, and
223 static int fifo_write(struct xillyfifo *fifo,
224 const void *data, unsigned int len,
225 int (*copier)(void *, const void *, int))
227 unsigned int done = 0;
228 unsigned int todo = len;
230 unsigned int writepos = fifo->writepos;
231 unsigned int writebuf = fifo->writebuf;
235 nmax = fifo->size - READ_ONCE(fifo->fill);
238 unsigned int nrail = fifo->bufsize - writepos;
239 unsigned int n = min(todo, nmax);
242 spin_lock_irqsave(&fifo->lock, flags);
244 spin_unlock_irqrestore(&fifo->lock, flags);
246 fifo->writepos = writepos;
247 fifo->writebuf = writebuf;
255 rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);
266 if (writepos == fifo->bufsize) {
270 if (writebuf == fifo->bufnum)
276 static int fifo_read(struct xillyfifo *fifo,
277 void *data, unsigned int len,
278 int (*copier)(void *, const void *, int))
280 unsigned int done = 0;
281 unsigned int todo = len;
283 unsigned int readpos = fifo->readpos;
284 unsigned int readbuf = fifo->readbuf;
289 * The spinlock here is necessary, because otherwise fifo->fill
290 * could have been increased by fifo_write() after writing data
291 * to the buffer, but this data would potentially not have been
292 * visible on this thread at the time the updated fifo->fill was.
293 * That could lead to reading invalid data.
296 spin_lock_irqsave(&fifo->lock, flags);
298 spin_unlock_irqrestore(&fifo->lock, flags);
301 unsigned int nrail = fifo->bufsize - readpos;
302 unsigned int n = min(todo, fill);
305 spin_lock_irqsave(&fifo->lock, flags);
307 spin_unlock_irqrestore(&fifo->lock, flags);
309 fifo->readpos = readpos;
310 fifo->readbuf = readbuf;
318 rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);
329 if (readpos == fifo->bufsize) {
333 if (readbuf == fifo->bufnum)
340 * These three wrapper functions are used as the @copier argument to
341 * fifo_write() and fifo_read(), so that they can work directly with
342 * user memory as well.
345 static int xilly_copy_from_user(void *dst, const void *src, int n)
347 if (copy_from_user(dst, (const void __user *)src, n))
353 static int xilly_copy_to_user(void *dst, const void *src, int n)
355 if (copy_to_user((void __user *)dst, src, n))
361 static int xilly_memcpy(void *dst, const void *src, int n)
368 static int fifo_init(struct xillyfifo *fifo,
369 unsigned int log2_size)
371 unsigned int log2_bufnum;
372 unsigned int buf_order;
375 unsigned int log2_fifo_buf_size;
378 log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT;
380 if (log2_size > log2_fifo_buf_size) {
381 log2_bufnum = log2_size - log2_fifo_buf_size;
382 buf_order = fifo_buf_order;
383 fifo->bufsize = 1 << log2_fifo_buf_size;
386 buf_order = (log2_size > PAGE_SHIFT) ?
387 log2_size - PAGE_SHIFT : 0;
388 fifo->bufsize = 1 << log2_size;
391 fifo->bufnum = 1 << log2_bufnum;
392 fifo->size = fifo->bufnum * fifo->bufsize;
393 fifo->buf_order = buf_order;
395 fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);
400 for (i = 0; i < fifo->bufnum; i++) {
401 fifo->mem[i] = (void *)
402 __get_free_pages(GFP_KERNEL, buf_order);
413 spin_lock_init(&fifo->lock);
414 init_waitqueue_head(&fifo->waitq);
418 for (i--; i >= 0; i--)
419 free_pages((unsigned long)fifo->mem[i], buf_order);
424 if (fifo_buf_order) {
432 static void fifo_mem_release(struct xillyfifo *fifo)
439 for (i = 0; i < fifo->bufnum; i++)
440 free_pages((unsigned long)fifo->mem[i], fifo->buf_order);
446 * When endpoint_quiesce() returns, the endpoint has no URBs submitted,
447 * won't accept any new URB submissions, and its related work item doesn't
448 * and won't run anymore.
451 static void endpoint_quiesce(struct xillyusb_endpoint *ep)
453 mutex_lock(&ep->ep_mutex);
454 ep->shutting_down = true;
455 mutex_unlock(&ep->ep_mutex);
457 usb_kill_anchored_urbs(&ep->anchor);
458 cancel_work_sync(&ep->workitem);
462 * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
463 * though endpoint_alloc doesn't allocate that memory.
466 static void endpoint_dealloc(struct xillyusb_endpoint *ep)
468 struct list_head *this, *next;
470 fifo_mem_release(&ep->fifo);
472 /* Join @filled_buffers with @buffers to free these entries too */
473 list_splice(&ep->filled_buffers, &ep->buffers);
475 list_for_each_safe(this, next, &ep->buffers) {
476 struct xillybuffer *xb =
477 list_entry(this, struct xillybuffer, entry);
479 free_pages((unsigned long)xb->buf, ep->order);
486 static struct xillyusb_endpoint
487 *endpoint_alloc(struct xillyusb_dev *xdev,
489 void (*work)(struct work_struct *),
495 struct xillyusb_endpoint *ep;
497 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
502 INIT_LIST_HEAD(&ep->buffers);
503 INIT_LIST_HEAD(&ep->filled_buffers);
505 spin_lock_init(&ep->buffers_lock);
506 mutex_init(&ep->ep_mutex);
508 init_usb_anchor(&ep->anchor);
509 INIT_WORK(&ep->workitem, work);
512 ep->buffer_size = 1 << (PAGE_SHIFT + order);
513 ep->outstanding_urbs = 0;
515 ep->wake_on_drain = false;
518 ep->shutting_down = false;
520 for (i = 0; i < bufnum; i++) {
521 struct xillybuffer *xb;
524 xb = kzalloc(sizeof(*xb), GFP_KERNEL);
527 endpoint_dealloc(ep);
531 addr = __get_free_pages(GFP_KERNEL, order);
535 endpoint_dealloc(ep);
539 xb->buf = (void *)addr;
541 list_add_tail(&xb->entry, &ep->buffers);
546 static void cleanup_dev(struct kref *kref)
548 struct xillyusb_dev *xdev =
549 container_of(kref, struct xillyusb_dev, kref);
552 endpoint_dealloc(xdev->in_ep);
555 endpoint_dealloc(xdev->msg_ep);
558 destroy_workqueue(xdev->workq);
560 usb_put_dev(xdev->udev);
561 kfree(xdev->channels); /* Argument may be NULL, and that's fine */
566 * @process_in_mutex is taken to ensure that bulk_in_work() won't call
567 * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all
568 * @read_data_ok entries, which will make process_bulk_in() report false
569 * errors if executed. The mechanism relies on that xdev->error is assigned
570 * a non-zero value by report_io_error() prior to queueing wakeup_all(),
571 * which prevents bulk_in_work() from calling process_bulk_in().
573 * The fact that wakeup_all() and bulk_in_work() are queued on the same
574 * workqueue makes their concurrent execution very unlikely, however the
575 * kernel's API doesn't seem to ensure this strictly.
578 static void wakeup_all(struct work_struct *work)
581 struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev,
584 mutex_lock(&xdev->process_in_mutex);
586 for (i = 0; i < xdev->num_channels; i++) {
587 struct xillyusb_channel *chan = &xdev->channels[i];
589 mutex_lock(&chan->lock);
593 * Fake an EOF: Even if such arrives, it won't be
596 chan->read_data_ok = 0;
597 wake_up_interruptible(&chan->in_fifo->waitq);
601 wake_up_interruptible(&chan->out_ep->fifo.waitq);
603 mutex_unlock(&chan->lock);
605 wake_up_interruptible(&chan->flushq);
608 mutex_unlock(&xdev->process_in_mutex);
610 wake_up_interruptible(&xdev->msg_ep->fifo.waitq);
612 kref_put(&xdev->kref, cleanup_dev);
615 static void report_io_error(struct xillyusb_dev *xdev,
619 bool do_once = false;
621 spin_lock_irqsave(&xdev->error_lock, flags);
623 xdev->error = errcode;
626 spin_unlock_irqrestore(&xdev->error_lock, flags);
629 kref_get(&xdev->kref); /* xdev is used by work item */
630 queue_work(xdev->workq, &xdev->wakeup_workitem);
635 * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures
636 * the previous pointer is never used after its return.
639 static void safely_assign_in_fifo(struct xillyusb_channel *chan,
640 struct xillyfifo *fifo)
642 mutex_lock(&chan->lock);
643 chan->in_fifo = fifo;
644 mutex_unlock(&chan->lock);
646 flush_work(&chan->xdev->in_ep->workitem);
649 static void bulk_in_completer(struct urb *urb)
651 struct xillybuffer *xb = urb->context;
652 struct xillyusb_endpoint *ep = xb->ep;
656 if (!(urb->status == -ENOENT ||
657 urb->status == -ECONNRESET ||
658 urb->status == -ESHUTDOWN))
659 report_io_error(ep->xdev, -EIO);
661 spin_lock_irqsave(&ep->buffers_lock, flags);
662 list_add_tail(&xb->entry, &ep->buffers);
663 ep->outstanding_urbs--;
664 spin_unlock_irqrestore(&ep->buffers_lock, flags);
669 xb->len = urb->actual_length;
671 spin_lock_irqsave(&ep->buffers_lock, flags);
672 list_add_tail(&xb->entry, &ep->filled_buffers);
673 spin_unlock_irqrestore(&ep->buffers_lock, flags);
675 if (!ep->shutting_down)
676 queue_work(ep->xdev->workq, &ep->workitem);
679 static void bulk_out_completer(struct urb *urb)
681 struct xillybuffer *xb = urb->context;
682 struct xillyusb_endpoint *ep = xb->ep;
686 (!(urb->status == -ENOENT ||
687 urb->status == -ECONNRESET ||
688 urb->status == -ESHUTDOWN)))
689 report_io_error(ep->xdev, -EIO);
691 spin_lock_irqsave(&ep->buffers_lock, flags);
692 list_add_tail(&xb->entry, &ep->buffers);
693 ep->outstanding_urbs--;
694 spin_unlock_irqrestore(&ep->buffers_lock, flags);
696 if (!ep->shutting_down)
697 queue_work(ep->xdev->workq, &ep->workitem);
700 static void try_queue_bulk_in(struct xillyusb_endpoint *ep)
702 struct xillyusb_dev *xdev = ep->xdev;
703 struct xillybuffer *xb;
708 unsigned int bufsize = ep->buffer_size;
710 mutex_lock(&ep->ep_mutex);
712 if (ep->shutting_down || xdev->error)
716 spin_lock_irqsave(&ep->buffers_lock, flags);
718 if (list_empty(&ep->buffers)) {
719 spin_unlock_irqrestore(&ep->buffers_lock, flags);
723 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
724 list_del(&xb->entry);
725 ep->outstanding_urbs++;
727 spin_unlock_irqrestore(&ep->buffers_lock, flags);
729 urb = usb_alloc_urb(0, GFP_KERNEL);
731 report_io_error(xdev, -ENOMEM);
735 usb_fill_bulk_urb(urb, xdev->udev,
736 usb_rcvbulkpipe(xdev->udev, ep->ep_num),
737 xb->buf, bufsize, bulk_in_completer, xb);
739 usb_anchor_urb(urb, &ep->anchor);
741 rc = usb_submit_urb(urb, GFP_KERNEL);
744 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
749 usb_free_urb(urb); /* This just decrements reference count */
753 usb_unanchor_urb(urb);
757 spin_lock_irqsave(&ep->buffers_lock, flags);
758 list_add_tail(&xb->entry, &ep->buffers);
759 ep->outstanding_urbs--;
760 spin_unlock_irqrestore(&ep->buffers_lock, flags);
763 mutex_unlock(&ep->ep_mutex);
766 static void try_queue_bulk_out(struct xillyusb_endpoint *ep)
768 struct xillyfifo *fifo = &ep->fifo;
769 struct xillyusb_dev *xdev = ep->xdev;
770 struct xillybuffer *xb;
776 bool do_wake = false;
778 mutex_lock(&ep->ep_mutex);
780 if (ep->shutting_down || xdev->error)
783 fill = READ_ONCE(fifo->fill) & ep->fill_mask;
787 unsigned int max_read;
789 spin_lock_irqsave(&ep->buffers_lock, flags);
792 * Race conditions might have the FIFO filled while the
793 * endpoint is marked as drained here. That doesn't matter,
794 * because the sole purpose of @drained is to ensure that
795 * certain data has been sent on the USB channel before
796 * shutting it down. Hence knowing that the FIFO appears
797 * to be empty with no outstanding URBs at some moment
802 ep->drained = !ep->outstanding_urbs;
803 if (ep->drained && ep->wake_on_drain)
806 spin_unlock_irqrestore(&ep->buffers_lock, flags);
812 if ((fill < ep->buffer_size && ep->outstanding_urbs) ||
813 list_empty(&ep->buffers)) {
814 spin_unlock_irqrestore(&ep->buffers_lock, flags);
818 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
819 list_del(&xb->entry);
820 ep->outstanding_urbs++;
822 spin_unlock_irqrestore(&ep->buffers_lock, flags);
824 max_read = min(fill, ep->buffer_size);
826 count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);
829 * xilly_memcpy always returns 0 => fifo_read can't fail =>
833 urb = usb_alloc_urb(0, GFP_KERNEL);
835 report_io_error(xdev, -ENOMEM);
839 usb_fill_bulk_urb(urb, xdev->udev,
840 usb_sndbulkpipe(xdev->udev, ep->ep_num),
841 xb->buf, count, bulk_out_completer, xb);
843 usb_anchor_urb(urb, &ep->anchor);
845 rc = usb_submit_urb(urb, GFP_KERNEL);
848 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
853 usb_free_urb(urb); /* This just decrements reference count */
860 usb_unanchor_urb(urb);
864 spin_lock_irqsave(&ep->buffers_lock, flags);
865 list_add_tail(&xb->entry, &ep->buffers);
866 ep->outstanding_urbs--;
867 spin_unlock_irqrestore(&ep->buffers_lock, flags);
870 mutex_unlock(&ep->ep_mutex);
873 wake_up_interruptible(&fifo->waitq);
876 static void bulk_out_work(struct work_struct *work)
878 struct xillyusb_endpoint *ep = container_of(work,
879 struct xillyusb_endpoint,
881 try_queue_bulk_out(ep);
884 static int process_in_opcode(struct xillyusb_dev *xdev,
888 struct xillyusb_channel *chan;
889 struct device *dev = xdev->dev;
890 int chan_idx = chan_num >> 1;
892 if (chan_idx >= xdev->num_channels) {
893 dev_err(dev, "Received illegal channel ID %d from FPGA\n",
898 chan = &xdev->channels[chan_idx];
902 if (!chan->read_data_ok) {
903 dev_err(dev, "Received unexpected EOF for channel %d\n",
909 * A write memory barrier ensures that the FIFO's fill level
910 * is visible before read_data_ok turns zero, so the data in
911 * the FIFO isn't missed by the consumer.
914 WRITE_ONCE(chan->read_data_ok, 0);
915 wake_up_interruptible(&chan->in_fifo->waitq);
918 case OPCODE_REACHED_CHECKPOINT:
920 wake_up_interruptible(&chan->flushq);
923 case OPCODE_CANCELED_CHECKPOINT:
925 wake_up_interruptible(&chan->flushq);
929 dev_err(dev, "Received illegal opcode %d from FPGA\n",
937 static int process_bulk_in(struct xillybuffer *xb)
939 struct xillyusb_endpoint *ep = xb->ep;
940 struct xillyusb_dev *xdev = ep->xdev;
941 struct device *dev = xdev->dev;
942 int dws = xb->len >> 2;
945 struct xillyusb_channel *chan;
946 struct xillyfifo *fifo;
947 int chan_num = 0, opcode;
949 int bytes, count, dwconsume;
950 int in_bytes_left = 0;
953 if ((dws << 2) != xb->len) {
954 dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n",
959 if (xdev->in_bytes_left) {
960 bytes = min(xdev->in_bytes_left, dws << 2);
961 in_bytes_left = xdev->in_bytes_left - bytes;
962 chan_num = xdev->leftover_chan_num;
963 goto resume_leftovers;
967 ctrlword = le32_to_cpu(*p++);
970 chan_num = ctrlword & 0xfff;
971 count = (ctrlword >> 12) & 0x3ff;
972 opcode = (ctrlword >> 24) & 0xf;
974 if (opcode != OPCODE_DATA) {
975 unsigned int in_counter = xdev->in_counter++ & 0x3ff;
977 if (count != in_counter) {
978 dev_err(dev, "Expected opcode counter %d, got %d\n",
983 rc = process_in_opcode(xdev, opcode, chan_num);
991 bytes = min(count + 1, dws << 2);
992 in_bytes_left = count + 1 - bytes;
995 chan_idx = chan_num >> 1;
997 if (!(chan_num & 1) || chan_idx >= xdev->num_channels ||
998 !xdev->channels[chan_idx].read_data_ok) {
999 dev_err(dev, "Received illegal channel ID %d from FPGA\n",
1003 chan = &xdev->channels[chan_idx];
1005 fifo = chan->in_fifo;
1007 if (unlikely(!fifo))
1008 return -EIO; /* We got really unexpected data */
1010 if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {
1011 dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");
1015 wake_up_interruptible(&fifo->waitq);
1017 dwconsume = (bytes + 3) >> 2;
1022 xdev->in_bytes_left = in_bytes_left;
1023 xdev->leftover_chan_num = chan_num;
1027 static void bulk_in_work(struct work_struct *work)
1029 struct xillyusb_endpoint *ep =
1030 container_of(work, struct xillyusb_endpoint, workitem);
1031 struct xillyusb_dev *xdev = ep->xdev;
1032 unsigned long flags;
1033 struct xillybuffer *xb;
1034 bool consumed = false;
1037 mutex_lock(&xdev->process_in_mutex);
1039 spin_lock_irqsave(&ep->buffers_lock, flags);
1042 if (rc || list_empty(&ep->filled_buffers)) {
1043 spin_unlock_irqrestore(&ep->buffers_lock, flags);
1044 mutex_unlock(&xdev->process_in_mutex);
1047 report_io_error(xdev, rc);
1049 try_queue_bulk_in(ep);
1054 xb = list_first_entry(&ep->filled_buffers, struct xillybuffer,
1056 list_del(&xb->entry);
1058 spin_unlock_irqrestore(&ep->buffers_lock, flags);
1063 rc = process_bulk_in(xb);
1065 spin_lock_irqsave(&ep->buffers_lock, flags);
1066 list_add_tail(&xb->entry, &ep->buffers);
1067 ep->outstanding_urbs--;
1071 static int xillyusb_send_opcode(struct xillyusb_dev *xdev,
1072 int chan_num, char opcode, u32 data)
1074 struct xillyusb_endpoint *ep = xdev->msg_ep;
1075 struct xillyfifo *fifo = &ep->fifo;
1080 msg[0] = cpu_to_le32((chan_num & 0xfff) |
1081 ((opcode & 0xf) << 24));
1082 msg[1] = cpu_to_le32(data);
1084 mutex_lock(&xdev->msg_mutex);
1087 * The wait queue is woken with the interruptible variant, so the
1088 * wait function matches, however returning because of an interrupt
1089 * will mess things up considerably, in particular when the caller is
1090 * the release method. And the xdev->error part prevents being stuck
1091 * forever in the event of a bizarre hardware bug: Pull the USB plug.
1094 while (wait_event_interruptible(fifo->waitq,
1095 fifo->fill <= (fifo->size - 8) ||
1104 fifo_write(fifo, (void *)msg, 8, xilly_memcpy);
1106 try_queue_bulk_out(ep);
1109 mutex_unlock(&xdev->msg_mutex);
1115 * Note that flush_downstream() merely waits for the data to arrive to
1116 * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,
1117 * it does nothing to make it happen (and neither is it necessary).
1119 * This function is not reentrant for the same @chan, but this is covered
1120 * by the fact that for any given @chan, it's called either by the open,
1121 * write, llseek and flush fops methods, which can't run in parallel (and the
1122 * write + flush and llseek method handlers are protected with out_mutex).
1124 * chan->flushed is there to avoid multiple flushes at the same position,
1125 * in particular as a result of programs that close the file descriptor
1126 * e.g. after a dup2() for redirection.
1129 static int flush_downstream(struct xillyusb_channel *chan,
1133 struct xillyusb_dev *xdev = chan->xdev;
1134 int chan_num = chan->chan_idx << 1;
1135 long deadline, left_to_sleep;
1141 deadline = jiffies + 1 + timeout;
1143 if (chan->flushing) {
1144 long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT;
1147 rc = xillyusb_send_opcode(xdev, chan_num,
1148 OPCODE_CANCEL_CHECKPOINT, 0);
1151 return rc; /* Only real error, never -EINTR */
1153 /* Ignoring interrupts. Cancellation must be handled */
1154 while (!chan->canceled) {
1155 left_to_sleep = cancel_deadline - ((long)jiffies);
1157 if (left_to_sleep <= 0) {
1158 report_io_error(xdev, -EIO);
1162 rc = wait_event_interruptible_timeout(chan->flushq,
1175 * The checkpoint is given in terms of data elements, not bytes. As
1176 * a result, if less than an element's worth of data is stored in the
1177 * FIFO, it's not flushed, including the flush before closing, which
1178 * means that such data is lost. This is consistent with PCIe Xillybus.
1181 rc = xillyusb_send_opcode(xdev, chan_num,
1182 OPCODE_SET_CHECKPOINT,
1184 chan->out_log2_element_size);
1187 return rc; /* Only real error, never -EINTR */
1190 while (chan->flushing) {
1191 rc = wait_event_interruptible(chan->flushq,
1197 if (interruptible && rc)
1204 while (chan->flushing) {
1205 left_to_sleep = deadline - ((long)jiffies);
1207 if (left_to_sleep <= 0)
1210 rc = wait_event_interruptible_timeout(chan->flushq,
1218 if (interruptible && rc < 0)
1227 /* request_read_anything(): Ask the FPGA for any little amount of data */
1228 static int request_read_anything(struct xillyusb_channel *chan,
1231 struct xillyusb_dev *xdev = chan->xdev;
1232 unsigned int sh = chan->in_log2_element_size;
1233 int chan_num = (chan->chan_idx << 1) | 1;
1234 u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1;
1236 return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh);
1239 static int xillyusb_open(struct inode *inode, struct file *filp)
1241 struct xillyusb_dev *xdev;
1242 struct xillyusb_channel *chan;
1243 struct xillyfifo *in_fifo = NULL;
1244 struct xillyusb_endpoint *out_ep = NULL;
1248 mutex_lock(&kref_mutex);
1250 rc = xillybus_find_inode(inode, (void **)&xdev, &index);
1252 mutex_unlock(&kref_mutex);
1256 kref_get(&xdev->kref);
1257 mutex_unlock(&kref_mutex);
1259 chan = &xdev->channels[index];
1260 filp->private_data = chan;
1262 mutex_lock(&chan->lock);
1269 if (((filp->f_mode & FMODE_READ) && !chan->readable) ||
1270 ((filp->f_mode & FMODE_WRITE) && !chan->writable))
1273 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) &&
1274 chan->in_synchronous) {
1276 "open() failed: O_NONBLOCK not allowed for read on this device\n");
1280 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) &&
1281 chan->out_synchronous) {
1283 "open() failed: O_NONBLOCK not allowed for write on this device\n");
1289 if (((filp->f_mode & FMODE_READ) && chan->open_for_read) ||
1290 ((filp->f_mode & FMODE_WRITE) && chan->open_for_write))
1293 if (filp->f_mode & FMODE_READ)
1294 chan->open_for_read = 1;
1296 if (filp->f_mode & FMODE_WRITE)
1297 chan->open_for_write = 1;
1299 mutex_unlock(&chan->lock);
1301 if (filp->f_mode & FMODE_WRITE) {
1302 out_ep = endpoint_alloc(xdev,
1303 (chan->chan_idx + 2) | USB_DIR_OUT,
1304 bulk_out_work, BUF_SIZE_ORDER, BUFNUM);
1311 rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);
1316 out_ep->fill_mask = -(1 << chan->out_log2_element_size);
1317 chan->out_bytes = 0;
1321 * Sending a flush request to a previously closed stream
1322 * effectively opens it, and also waits until the command is
1323 * confirmed by the FPGA. The latter is necessary because the
1324 * data is sent through a separate BULK OUT endpoint, and the
1325 * xHCI controller is free to reorder transmissions.
1327 * This can't go wrong unless there's a serious hardware error
1328 * (or the computer is stuck for 500 ms?)
1330 rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false);
1332 if (rc == -ETIMEDOUT) {
1334 report_io_error(xdev, rc);
1341 if (filp->f_mode & FMODE_READ) {
1342 in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL);
1349 rc = fifo_init(in_fifo, chan->in_log2_fifo_size);
1357 mutex_lock(&chan->lock);
1359 chan->in_fifo = in_fifo;
1360 chan->read_data_ok = 1;
1363 chan->out_ep = out_ep;
1364 mutex_unlock(&chan->lock);
1367 u32 in_checkpoint = 0;
1369 if (!chan->in_synchronous)
1370 in_checkpoint = in_fifo->size >>
1371 chan->in_log2_element_size;
1373 chan->in_consumed_bytes = 0;
1374 chan->poll_used = 0;
1375 chan->in_current_checkpoint = in_checkpoint;
1376 rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1377 OPCODE_SET_CHECKPOINT,
1380 if (rc) /* Failure guarantees that opcode wasn't sent */
1384 * In non-blocking mode, request the FPGA to send any data it
1385 * has right away. Otherwise, the first read() will always
1386 * return -EAGAIN, which is OK strictly speaking, but ugly.
1387 * Checking and unrolling if this fails isn't worth the
1388 * effort -- the error is propagated to the first read()
1391 if (filp->f_flags & O_NONBLOCK)
1392 request_read_anything(chan, OPCODE_SET_PUSH);
1398 chan->read_data_ok = 0;
1399 safely_assign_in_fifo(chan, NULL);
1400 fifo_mem_release(in_fifo);
1404 mutex_lock(&chan->lock);
1405 chan->out_ep = NULL;
1406 mutex_unlock(&chan->lock);
1411 endpoint_dealloc(out_ep);
1414 mutex_lock(&chan->lock);
1416 if (filp->f_mode & FMODE_READ)
1417 chan->open_for_read = 0;
1419 if (filp->f_mode & FMODE_WRITE)
1420 chan->open_for_write = 0;
1422 mutex_unlock(&chan->lock);
1424 kref_put(&xdev->kref, cleanup_dev);
1429 kref_put(&xdev->kref, cleanup_dev);
1430 mutex_unlock(&chan->lock);
1434 static ssize_t xillyusb_read(struct file *filp, char __user *userbuf,
1435 size_t count, loff_t *f_pos)
1437 struct xillyusb_channel *chan = filp->private_data;
1438 struct xillyusb_dev *xdev = chan->xdev;
1439 struct xillyfifo *fifo = chan->in_fifo;
1440 int chan_num = (chan->chan_idx << 1) | 1;
1442 long deadline, left_to_sleep;
1444 bool sent_set_push = false;
1447 deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
1449 rc = mutex_lock_interruptible(&chan->in_mutex);
1455 u32 fifo_checkpoint_bytes, complete_checkpoint_bytes;
1456 u32 complete_checkpoint, fifo_checkpoint;
1459 unsigned int sh = chan->in_log2_element_size;
1460 bool checkpoint_for_complete;
1462 rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,
1463 count - bytes_done, xilly_copy_to_user);
1469 chan->in_consumed_bytes += rc;
1471 left_to_sleep = deadline - ((long)jiffies);
1474 * Some 32-bit arithmetic that may wrap. Note that
1475 * complete_checkpoint is rounded up to the closest element
1476 * boundary, because the read() can't be completed otherwise.
1477 * fifo_checkpoint_bytes is rounded down, because it protects
1478 * in_fifo from overflowing.
1481 fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;
1482 complete_checkpoint_bytes =
1483 chan->in_consumed_bytes + count - bytes_done;
1485 fifo_checkpoint = fifo_checkpoint_bytes >> sh;
1486 complete_checkpoint =
1487 (complete_checkpoint_bytes + (1 << sh) - 1) >> sh;
1489 diff = (fifo_checkpoint - complete_checkpoint) << sh;
1491 if (chan->in_synchronous && diff >= 0) {
1492 checkpoint = complete_checkpoint;
1493 checkpoint_for_complete = true;
1495 checkpoint = fifo_checkpoint;
1496 checkpoint_for_complete = false;
1499 leap = (checkpoint - chan->in_current_checkpoint) << sh;
1502 * To prevent flooding of OPCODE_SET_CHECKPOINT commands as
1503 * data is consumed, it's issued only if it moves the
1504 * checkpoint by at least an 8th of the FIFO's size, or if
1505 * it's necessary to complete the number of bytes requested by
1508 * chan->read_data_ok is checked to spare an unnecessary
1509 * submission after receiving EOF, however it's harmless if
1513 if (chan->read_data_ok &&
1514 (leap > (fifo->size >> 3) ||
1515 (checkpoint_for_complete && leap > 0))) {
1516 chan->in_current_checkpoint = checkpoint;
1517 rc = xillyusb_send_opcode(xdev, chan_num,
1518 OPCODE_SET_CHECKPOINT,
1525 if (bytes_done == count ||
1526 (left_to_sleep <= 0 && bytes_done))
1530 * Reaching here means that the FIFO was empty when
1531 * fifo_read() returned, but not necessarily right now. Error
1532 * and EOF are checked and reported only now, so that no data
1533 * that managed its way to the FIFO is lost.
1536 if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */
1537 /* Has data slipped into the FIFO since fifo_read()? */
1539 if (READ_ONCE(fifo->fill))
1551 if (filp->f_flags & O_NONBLOCK) {
1556 if (!sent_set_push) {
1557 rc = xillyusb_send_opcode(xdev, chan_num,
1559 complete_checkpoint);
1564 sent_set_push = true;
1567 if (left_to_sleep > 0) {
1569 * Note that when xdev->error is set (e.g. when the
1570 * device is unplugged), read_data_ok turns zero and
1571 * fifo->waitq is awaken.
1572 * Therefore no special attention to xdev->error.
1575 rc = wait_event_interruptible_timeout
1577 fifo->fill || !chan->read_data_ok,
1579 } else { /* bytes_done == 0 */
1580 /* Tell FPGA to send anything it has */
1581 rc = request_read_anything(chan, OPCODE_UPDATE_PUSH);
1586 rc = wait_event_interruptible
1588 fifo->fill || !chan->read_data_ok);
1597 if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) &&
1598 !READ_ONCE(fifo->fill))
1599 request_read_anything(chan, OPCODE_SET_PUSH);
1601 mutex_unlock(&chan->in_mutex);
1609 static int xillyusb_flush(struct file *filp, fl_owner_t id)
1611 struct xillyusb_channel *chan = filp->private_data;
1614 if (!(filp->f_mode & FMODE_WRITE))
1617 rc = mutex_lock_interruptible(&chan->out_mutex);
1623 * One second's timeout on flushing. Interrupts are ignored, because if
1624 * the user pressed CTRL-C, that interrupt will still be in flight by
1625 * the time we reach here, and the opportunity to flush is lost.
1627 rc = flush_downstream(chan, HZ, false);
1629 mutex_unlock(&chan->out_mutex);
1631 if (rc == -ETIMEDOUT) {
1632 /* The things you do to use dev_warn() and not pr_warn() */
1633 struct xillyusb_dev *xdev = chan->xdev;
1635 mutex_lock(&chan->lock);
1638 "Timed out while flushing. Output data may be lost.\n");
1639 mutex_unlock(&chan->lock);
1645 static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf,
1646 size_t count, loff_t *f_pos)
1648 struct xillyusb_channel *chan = filp->private_data;
1649 struct xillyusb_dev *xdev = chan->xdev;
1650 struct xillyfifo *fifo = &chan->out_ep->fifo;
1653 rc = mutex_lock_interruptible(&chan->out_mutex);
1667 rc = fifo_write(fifo, (__force void *)userbuf, count,
1668 xilly_copy_from_user);
1673 if (filp->f_flags & O_NONBLOCK) {
1678 if (wait_event_interruptible
1680 fifo->fill != fifo->size || xdev->error)) {
1689 chan->out_bytes += rc;
1692 try_queue_bulk_out(chan->out_ep);
1696 if (chan->out_synchronous) {
1697 int flush_rc = flush_downstream(chan, 0, true);
1699 if (flush_rc && !rc)
1704 mutex_unlock(&chan->out_mutex);
1709 static int xillyusb_release(struct inode *inode, struct file *filp)
1711 struct xillyusb_channel *chan = filp->private_data;
1712 struct xillyusb_dev *xdev = chan->xdev;
1713 int rc_read = 0, rc_write = 0;
1715 if (filp->f_mode & FMODE_READ) {
1716 struct xillyfifo *in_fifo = chan->in_fifo;
1718 rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1721 * If rc_read is nonzero, xdev->error indicates a global
1722 * device error. The error is reported later, so that
1723 * resources are freed.
1725 * Looping on wait_event_interruptible() kinda breaks the idea
1726 * of being interruptible, and this should have been
1727 * wait_event(). Only it's being waken with
1728 * wake_up_interruptible() for the sake of other uses. If
1729 * there's a global device error, chan->read_data_ok is
1730 * deasserted and the wait queue is awaken, so this is covered.
1733 while (wait_event_interruptible(in_fifo->waitq,
1734 !chan->read_data_ok))
1737 safely_assign_in_fifo(chan, NULL);
1738 fifo_mem_release(in_fifo);
1741 mutex_lock(&chan->lock);
1742 chan->open_for_read = 0;
1743 mutex_unlock(&chan->lock);
1746 if (filp->f_mode & FMODE_WRITE) {
1747 struct xillyusb_endpoint *ep = chan->out_ep;
1749 * chan->flushing isn't zeroed. If the pre-release flush timed
1750 * out, a cancel request will be sent before the next
1751 * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again).
1752 * This is despite that the FPGA forgets about the checkpoint
1753 * request as the file closes. Still, in an exceptional race
1754 * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT
1755 * just before closing that would reach the host after the
1756 * file has re-opened.
1759 mutex_lock(&chan->lock);
1760 chan->out_ep = NULL;
1761 mutex_unlock(&chan->lock);
1763 endpoint_quiesce(ep);
1764 endpoint_dealloc(ep);
1766 /* See comments on rc_read above */
1767 rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1770 mutex_lock(&chan->lock);
1771 chan->open_for_write = 0;
1772 mutex_unlock(&chan->lock);
1775 kref_put(&xdev->kref, cleanup_dev);
1777 return rc_read ? rc_read : rc_write;
1781 * Xillybus' API allows device nodes to be seekable, giving the user
1782 * application access to a RAM array on the FPGA (or logic emulating it).
1785 static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence)
1787 struct xillyusb_channel *chan = filp->private_data;
1788 struct xillyusb_dev *xdev = chan->xdev;
1789 loff_t pos = filp->f_pos;
1791 unsigned int log2_element_size = chan->readable ?
1792 chan->in_log2_element_size : chan->out_log2_element_size;
1795 * Take both mutexes not allowing interrupts, since it seems like
1796 * common applications don't expect an -EINTR here. Besides, multiple
1797 * access to a single file descriptor on seekable devices is a mess
1801 mutex_lock(&chan->out_mutex);
1802 mutex_lock(&chan->in_mutex);
1812 pos = offset; /* Going to the end => to the beginning */
1819 /* In any case, we must finish on an element boundary */
1820 if (pos & ((1 << log2_element_size) - 1)) {
1825 rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1827 pos >> log2_element_size);
1832 if (chan->writable) {
1834 rc = flush_downstream(chan, HZ, false);
1838 mutex_unlock(&chan->out_mutex);
1839 mutex_unlock(&chan->in_mutex);
1841 if (rc) /* Return error after releasing mutexes */
1849 static __poll_t xillyusb_poll(struct file *filp, poll_table *wait)
1851 struct xillyusb_channel *chan = filp->private_data;
1855 poll_wait(filp, &chan->in_fifo->waitq, wait);
1858 poll_wait(filp, &chan->out_ep->fifo.waitq, wait);
1861 * If this is the first time poll() is called, and the file is
1862 * readable, set the relevant flag. Also tell the FPGA to send all it
1863 * has, to kickstart the mechanism that ensures there's always some
1864 * data in in_fifo unless the stream is dry end-to-end. Note that the
1865 * first poll() may not return a EPOLLIN, even if there's data on the
1866 * FPGA. Rather, the data will arrive soon, and trigger the relevant
1870 if (!chan->poll_used && chan->in_fifo) {
1871 chan->poll_used = 1;
1872 request_read_anything(chan, OPCODE_SET_PUSH);
1876 * poll() won't play ball regarding read() channels which
1877 * are synchronous. Allowing that will create situations where data has
1878 * been delivered at the FPGA, and users expecting select() to wake up,
1879 * which it may not. So make it never work.
1882 if (chan->in_fifo && !chan->in_synchronous &&
1883 (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok))
1884 mask |= EPOLLIN | EPOLLRDNORM;
1887 (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))
1888 mask |= EPOLLOUT | EPOLLWRNORM;
1890 if (chan->xdev->error)
1896 static const struct file_operations xillyusb_fops = {
1897 .owner = THIS_MODULE,
1898 .read = xillyusb_read,
1899 .write = xillyusb_write,
1900 .open = xillyusb_open,
1901 .flush = xillyusb_flush,
1902 .release = xillyusb_release,
1903 .llseek = xillyusb_llseek,
1904 .poll = xillyusb_poll,
1907 static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)
1909 xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT,
1910 bulk_out_work, 1, 2);
1914 if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */
1917 xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */
1919 xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN,
1920 bulk_in_work, BUF_SIZE_ORDER, BUFNUM);
1924 try_queue_bulk_in(xdev->in_ep);
1929 endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */
1930 xdev->msg_ep = NULL;
1934 static int setup_channels(struct xillyusb_dev *xdev,
1938 struct xillyusb_channel *chan;
1941 chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL);
1945 xdev->channels = chan;
1947 for (i = 0; i < num_channels; i++, chan++) {
1948 unsigned int in_desc = le16_to_cpu(*chandesc++);
1949 unsigned int out_desc = le16_to_cpu(*chandesc++);
1952 mutex_init(&chan->in_mutex);
1953 mutex_init(&chan->out_mutex);
1954 mutex_init(&chan->lock);
1955 init_waitqueue_head(&chan->flushq);
1959 if (in_desc & 0x80) { /* Entry is valid */
1961 chan->in_synchronous = !!(in_desc & 0x40);
1962 chan->in_seekable = !!(in_desc & 0x20);
1963 chan->in_log2_element_size = in_desc & 0x0f;
1964 chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16;
1968 * A downstream channel should never exist above index 13,
1969 * as it would request a nonexistent BULK endpoint > 15.
1970 * In the peculiar case that it does, it's ignored silently.
1973 if ((out_desc & 0x80) && i < 14) { /* Entry is valid */
1975 chan->out_synchronous = !!(out_desc & 0x40);
1976 chan->out_seekable = !!(out_desc & 0x20);
1977 chan->out_log2_element_size = out_desc & 0x0f;
1978 chan->out_log2_fifo_size =
1979 ((out_desc >> 8) & 0x1f) + 16;
1986 static int xillyusb_discovery(struct usb_interface *interface)
1989 struct xillyusb_dev *xdev = usb_get_intfdata(interface);
1990 __le16 bogus_chandesc[2];
1991 struct xillyfifo idt_fifo;
1992 struct xillyusb_channel *chan;
1993 unsigned int idt_len, names_offset;
1997 rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2000 dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n");
2004 /* Phase I: Set up one fake upstream channel and obtain IDT */
2006 /* Set up a fake IDT with one async IN stream */
2007 bogus_chandesc[0] = cpu_to_le16(0x80);
2008 bogus_chandesc[1] = cpu_to_le16(0);
2010 rc = setup_channels(xdev, bogus_chandesc, 1);
2015 rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE);
2020 chan = xdev->channels;
2022 chan->in_fifo = &idt_fifo;
2023 chan->read_data_ok = 1;
2025 xdev->num_channels = 1;
2027 rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0);
2030 dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n");
2034 rc = wait_event_interruptible_timeout(idt_fifo.waitq,
2035 !chan->read_data_ok,
2036 XILLY_RESPONSE_TIMEOUT);
2044 rc = -EINTR; /* Interrupt on probe method? Interesting. */
2048 if (chan->read_data_ok) {
2050 dev_err(&interface->dev, "No response from FPGA. Aborting.\n");
2054 idt_len = READ_ONCE(idt_fifo.fill);
2055 idt = kmalloc(idt_len, GFP_KERNEL);
2062 fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy);
2064 if (crc32_le(~0, idt, idt_len) != 0) {
2065 dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n");
2071 dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n",
2077 /* Phase II: Set up the streams as defined in IDT */
2079 num_channels = le16_to_cpu(*((__le16 *)(idt + 1)));
2080 names_offset = 3 + num_channels * 4;
2081 idt_len -= 4; /* Exclude CRC */
2083 if (idt_len < names_offset) {
2084 dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n");
2089 rc = setup_channels(xdev, (void *)idt + 3, num_channels);
2095 * Except for wildly misbehaving hardware, or if it was disconnected
2096 * just after responding with the IDT, there is no reason for any
2097 * work item to be running now. To be sure that xdev->channels
2098 * is updated on anything that might run in parallel, flush the
2099 * workqueue, which rarely does anything.
2101 flush_workqueue(xdev->workq);
2103 xdev->num_channels = num_channels;
2105 fifo_mem_release(&idt_fifo);
2108 rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops,
2111 idt_len - names_offset,
2123 safely_assign_in_fifo(chan, NULL);
2124 fifo_mem_release(&idt_fifo);
2129 static int xillyusb_probe(struct usb_interface *interface,
2130 const struct usb_device_id *id)
2132 struct xillyusb_dev *xdev;
2135 xdev = kzalloc(sizeof(*xdev), GFP_KERNEL);
2139 kref_init(&xdev->kref);
2140 mutex_init(&xdev->process_in_mutex);
2141 mutex_init(&xdev->msg_mutex);
2143 xdev->udev = usb_get_dev(interface_to_usbdev(interface));
2144 xdev->dev = &interface->dev;
2146 spin_lock_init(&xdev->error_lock);
2147 xdev->in_counter = 0;
2148 xdev->in_bytes_left = 0;
2149 xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0);
2152 dev_err(&interface->dev, "Failed to allocate work queue\n");
2157 INIT_WORK(&xdev->wakeup_workitem, wakeup_all);
2159 usb_set_intfdata(interface, xdev);
2161 rc = xillyusb_setup_base_eps(xdev);
2165 rc = xillyusb_discovery(interface);
2172 endpoint_quiesce(xdev->in_ep);
2173 endpoint_quiesce(xdev->msg_ep);
2176 usb_set_intfdata(interface, NULL);
2177 kref_put(&xdev->kref, cleanup_dev);
2181 static void xillyusb_disconnect(struct usb_interface *interface)
2183 struct xillyusb_dev *xdev = usb_get_intfdata(interface);
2184 struct xillyusb_endpoint *msg_ep = xdev->msg_ep;
2185 struct xillyfifo *fifo = &msg_ep->fifo;
2189 xillybus_cleanup_chrdev(xdev, &interface->dev);
2192 * Try to send OPCODE_QUIESCE, which will fail silently if the device
2193 * was disconnected, but makes sense on module unload.
2196 msg_ep->wake_on_drain = true;
2197 xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2200 * If the device has been disconnected, sending the opcode causes
2201 * a global device error with xdev->error, if such error didn't
2202 * occur earlier. Hence timing out means that the USB link is fine,
2203 * but somehow the message wasn't sent. Should never happen.
2206 rc = wait_event_interruptible_timeout(fifo->waitq,
2207 msg_ep->drained || xdev->error,
2208 XILLY_RESPONSE_TIMEOUT);
2211 dev_err(&interface->dev,
2212 "Weird timeout condition on sending quiesce request.\n");
2214 report_io_error(xdev, -ENODEV); /* Discourage further activity */
2217 * This device driver is declared with soft_unbind set, or else
2218 * sending OPCODE_QUIESCE above would always fail. The price is
2219 * that the USB framework didn't kill outstanding URBs, so it has
2220 * to be done explicitly before returning from this call.
2223 for (i = 0; i < xdev->num_channels; i++) {
2224 struct xillyusb_channel *chan = &xdev->channels[i];
2227 * Lock taken to prevent chan->out_ep from changing. It also
2228 * ensures xillyusb_open() and xillyusb_flush() don't access
2229 * xdev->dev after being nullified below.
2231 mutex_lock(&chan->lock);
2233 endpoint_quiesce(chan->out_ep);
2234 mutex_unlock(&chan->lock);
2237 endpoint_quiesce(xdev->in_ep);
2238 endpoint_quiesce(xdev->msg_ep);
2240 usb_set_intfdata(interface, NULL);
2244 mutex_lock(&kref_mutex);
2245 kref_put(&xdev->kref, cleanup_dev);
2246 mutex_unlock(&kref_mutex);
2249 static struct usb_driver xillyusb_driver = {
2251 .id_table = xillyusb_table,
2252 .probe = xillyusb_probe,
2253 .disconnect = xillyusb_disconnect,
2257 static int __init xillyusb_init(void)
2261 if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT)
2262 fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT;
2266 rc = usb_register(&xillyusb_driver);
2271 static void __exit xillyusb_exit(void)
2273 usb_deregister(&xillyusb_driver);
2276 module_init(xillyusb_init);
2277 module_exit(xillyusb_exit);