1 // SPDX-License-Identifier: GPL-2.0-only
2 /* The industrial I/O core
4 * Copyright (c) 2008 Jonathan Cameron
6 * Handling of buffer allocation / resizing.
8 * Things to look at here.
9 * - Better memory allocation techniques?
10 * - Alternative access techniques?
12 #include <linux/kernel.h>
13 #include <linux/export.h>
14 #include <linux/device.h>
16 #include <linux/cdev.h>
17 #include <linux/slab.h>
18 #include <linux/poll.h>
19 #include <linux/sched/signal.h>
21 #include <linux/iio/iio.h>
22 #include <linux/iio/iio-opaque.h>
24 #include "iio_core_trigger.h"
25 #include <linux/iio/sysfs.h>
26 #include <linux/iio/buffer.h>
27 #include <linux/iio/buffer_impl.h>
29 static const char * const iio_endian_prefix[] = {
34 static bool iio_buffer_is_active(struct iio_buffer *buf)
36 return !list_empty(&buf->buffer_list);
39 static size_t iio_buffer_data_available(struct iio_buffer *buf)
41 return buf->access->data_available(buf);
44 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,
45 struct iio_buffer *buf, size_t required)
47 if (!indio_dev->info->hwfifo_flush_to_buffer)
50 return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);
53 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,
54 size_t to_wait, int to_flush)
59 /* wakeup if the device was unregistered */
63 /* drain the buffer if it was disabled */
64 if (!iio_buffer_is_active(buf)) {
65 to_wait = min_t(size_t, to_wait, 1);
69 avail = iio_buffer_data_available(buf);
71 if (avail >= to_wait) {
72 /* force a flush for non-blocking reads */
73 if (!to_wait && avail < to_flush)
74 iio_buffer_flush_hwfifo(indio_dev, buf,
80 flushed = iio_buffer_flush_hwfifo(indio_dev, buf,
85 if (avail + flushed >= to_wait)
92 * iio_buffer_read_outer() - chrdev read for buffer access
93 * @filp: File structure pointer for the char device
94 * @buf: Destination buffer for iio buffer read
95 * @n: First n bytes to read
96 * @f_ps: Long offset provided by the user as a seek position
98 * This function relies on all buffer implementations having an
99 * iio_buffer as their first element.
101 * Return: negative values corresponding to error codes or ret != 0
102 * for ending the reading activity
104 ssize_t iio_buffer_read_outer(struct file *filp, char __user *buf,
105 size_t n, loff_t *f_ps)
107 struct iio_dev *indio_dev = filp->private_data;
108 struct iio_buffer *rb = indio_dev->buffer;
109 DEFINE_WAIT_FUNC(wait, woken_wake_function);
114 if (!indio_dev->info)
117 if (!rb || !rb->access->read)
120 datum_size = rb->bytes_per_datum;
123 * If datum_size is 0 there will never be anything to read from the
124 * buffer, so signal end of file now.
129 if (filp->f_flags & O_NONBLOCK)
132 to_wait = min_t(size_t, n / datum_size, rb->watermark);
134 add_wait_queue(&rb->pollq, &wait);
136 if (!indio_dev->info) {
141 if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {
142 if (signal_pending(current)) {
147 wait_woken(&wait, TASK_INTERRUPTIBLE,
148 MAX_SCHEDULE_TIMEOUT);
152 ret = rb->access->read(rb, n, buf);
153 if (ret == 0 && (filp->f_flags & O_NONBLOCK))
156 remove_wait_queue(&rb->pollq, &wait);
162 * iio_buffer_poll() - poll the buffer to find out if it has data
163 * @filp: File structure pointer for device access
164 * @wait: Poll table structure pointer for which the driver adds
167 * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
168 * or 0 for other cases
170 __poll_t iio_buffer_poll(struct file *filp,
171 struct poll_table_struct *wait)
173 struct iio_dev *indio_dev = filp->private_data;
174 struct iio_buffer *rb = indio_dev->buffer;
176 if (!indio_dev->info || rb == NULL)
179 poll_wait(filp, &rb->pollq, wait);
180 if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))
181 return EPOLLIN | EPOLLRDNORM;
186 * iio_buffer_wakeup_poll - Wakes up the buffer waitqueue
187 * @indio_dev: The IIO device
189 * Wakes up the event waitqueue used for poll(). Should usually
190 * be called when the device is unregistered.
192 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)
194 struct iio_buffer *buffer = indio_dev->buffer;
199 wake_up(&buffer->pollq);
202 void iio_buffer_init(struct iio_buffer *buffer)
204 INIT_LIST_HEAD(&buffer->demux_list);
205 INIT_LIST_HEAD(&buffer->buffer_list);
206 init_waitqueue_head(&buffer->pollq);
207 kref_init(&buffer->ref);
208 if (!buffer->watermark)
209 buffer->watermark = 1;
211 EXPORT_SYMBOL(iio_buffer_init);
214 * iio_buffer_set_attrs - Set buffer specific attributes
215 * @buffer: The buffer for which we are setting attributes
216 * @attrs: Pointer to a null terminated list of pointers to attributes
218 void iio_buffer_set_attrs(struct iio_buffer *buffer,
219 const struct attribute **attrs)
221 buffer->attrs = attrs;
223 EXPORT_SYMBOL_GPL(iio_buffer_set_attrs);
225 static ssize_t iio_show_scan_index(struct device *dev,
226 struct device_attribute *attr,
229 return sprintf(buf, "%u\n", to_iio_dev_attr(attr)->c->scan_index);
232 static ssize_t iio_show_fixed_type(struct device *dev,
233 struct device_attribute *attr,
236 struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
237 u8 type = this_attr->c->scan_type.endianness;
239 if (type == IIO_CPU) {
240 #ifdef __LITTLE_ENDIAN
246 if (this_attr->c->scan_type.repeat > 1)
247 return sprintf(buf, "%s:%c%d/%dX%d>>%u\n",
248 iio_endian_prefix[type],
249 this_attr->c->scan_type.sign,
250 this_attr->c->scan_type.realbits,
251 this_attr->c->scan_type.storagebits,
252 this_attr->c->scan_type.repeat,
253 this_attr->c->scan_type.shift);
255 return sprintf(buf, "%s:%c%d/%d>>%u\n",
256 iio_endian_prefix[type],
257 this_attr->c->scan_type.sign,
258 this_attr->c->scan_type.realbits,
259 this_attr->c->scan_type.storagebits,
260 this_attr->c->scan_type.shift);
263 static ssize_t iio_scan_el_show(struct device *dev,
264 struct device_attribute *attr,
268 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
269 struct iio_buffer *buffer = indio_dev->buffer;
271 /* Ensure ret is 0 or 1. */
272 ret = !!test_bit(to_iio_dev_attr(attr)->address,
275 return sprintf(buf, "%d\n", ret);
278 /* Note NULL used as error indicator as it doesn't make sense. */
279 static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks,
280 unsigned int masklength,
281 const unsigned long *mask,
284 if (bitmap_empty(mask, masklength))
288 if (bitmap_equal(mask, av_masks, masklength))
291 if (bitmap_subset(mask, av_masks, masklength))
294 av_masks += BITS_TO_LONGS(masklength);
299 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,
300 const unsigned long *mask)
302 if (!indio_dev->setup_ops->validate_scan_mask)
305 return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);
309 * iio_scan_mask_set() - set particular bit in the scan mask
310 * @indio_dev: the iio device
311 * @buffer: the buffer whose scan mask we are interested in
312 * @bit: the bit to be set.
314 * Note that at this point we have no way of knowing what other
315 * buffers might request, hence this code only verifies that the
316 * individual buffers request is plausible.
318 static int iio_scan_mask_set(struct iio_dev *indio_dev,
319 struct iio_buffer *buffer, int bit)
321 const unsigned long *mask;
322 unsigned long *trialmask;
324 trialmask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);
325 if (trialmask == NULL)
327 if (!indio_dev->masklength) {
328 WARN(1, "Trying to set scanmask prior to registering buffer\n");
329 goto err_invalid_mask;
331 bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);
332 set_bit(bit, trialmask);
334 if (!iio_validate_scan_mask(indio_dev, trialmask))
335 goto err_invalid_mask;
337 if (indio_dev->available_scan_masks) {
338 mask = iio_scan_mask_match(indio_dev->available_scan_masks,
339 indio_dev->masklength,
342 goto err_invalid_mask;
344 bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);
346 bitmap_free(trialmask);
351 bitmap_free(trialmask);
355 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)
357 clear_bit(bit, buffer->scan_mask);
361 static int iio_scan_mask_query(struct iio_dev *indio_dev,
362 struct iio_buffer *buffer, int bit)
364 if (bit > indio_dev->masklength)
367 if (!buffer->scan_mask)
370 /* Ensure return value is 0 or 1. */
371 return !!test_bit(bit, buffer->scan_mask);
374 static ssize_t iio_scan_el_store(struct device *dev,
375 struct device_attribute *attr,
381 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
382 struct iio_buffer *buffer = indio_dev->buffer;
383 struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
385 ret = strtobool(buf, &state);
388 mutex_lock(&indio_dev->mlock);
389 if (iio_buffer_is_active(buffer)) {
393 ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);
397 ret = iio_scan_mask_clear(buffer, this_attr->address);
400 } else if (state && !ret) {
401 ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);
407 mutex_unlock(&indio_dev->mlock);
409 return ret < 0 ? ret : len;
413 static ssize_t iio_scan_el_ts_show(struct device *dev,
414 struct device_attribute *attr,
417 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
418 struct iio_buffer *buffer = indio_dev->buffer;
420 return sprintf(buf, "%d\n", buffer->scan_timestamp);
423 static ssize_t iio_scan_el_ts_store(struct device *dev,
424 struct device_attribute *attr,
429 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
430 struct iio_buffer *buffer = indio_dev->buffer;
433 ret = strtobool(buf, &state);
437 mutex_lock(&indio_dev->mlock);
438 if (iio_buffer_is_active(buffer)) {
442 buffer->scan_timestamp = state;
444 mutex_unlock(&indio_dev->mlock);
446 return ret ? ret : len;
449 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,
450 struct iio_buffer *buffer,
451 const struct iio_chan_spec *chan)
453 int ret, attrcount = 0;
455 ret = __iio_add_chan_devattr("index",
457 &iio_show_scan_index,
462 &buffer->scan_el_dev_attr_list);
466 ret = __iio_add_chan_devattr("type",
468 &iio_show_fixed_type,
473 &buffer->scan_el_dev_attr_list);
477 if (chan->type != IIO_TIMESTAMP)
478 ret = __iio_add_chan_devattr("en",
485 &buffer->scan_el_dev_attr_list);
487 ret = __iio_add_chan_devattr("en",
489 &iio_scan_el_ts_show,
490 &iio_scan_el_ts_store,
494 &buffer->scan_el_dev_attr_list);
502 static ssize_t iio_buffer_read_length(struct device *dev,
503 struct device_attribute *attr,
506 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
507 struct iio_buffer *buffer = indio_dev->buffer;
509 return sprintf(buf, "%d\n", buffer->length);
512 static ssize_t iio_buffer_write_length(struct device *dev,
513 struct device_attribute *attr,
514 const char *buf, size_t len)
516 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
517 struct iio_buffer *buffer = indio_dev->buffer;
521 ret = kstrtouint(buf, 10, &val);
525 if (val == buffer->length)
528 mutex_lock(&indio_dev->mlock);
529 if (iio_buffer_is_active(buffer)) {
532 buffer->access->set_length(buffer, val);
537 if (buffer->length && buffer->length < buffer->watermark)
538 buffer->watermark = buffer->length;
540 mutex_unlock(&indio_dev->mlock);
542 return ret ? ret : len;
545 static ssize_t iio_buffer_show_enable(struct device *dev,
546 struct device_attribute *attr,
549 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
550 struct iio_buffer *buffer = indio_dev->buffer;
552 return sprintf(buf, "%d\n", iio_buffer_is_active(buffer));
555 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
556 unsigned int scan_index)
558 const struct iio_chan_spec *ch;
561 ch = iio_find_channel_from_si(indio_dev, scan_index);
562 bytes = ch->scan_type.storagebits / 8;
563 if (ch->scan_type.repeat > 1)
564 bytes *= ch->scan_type.repeat;
568 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)
570 return iio_storage_bytes_for_si(indio_dev,
571 indio_dev->scan_index_timestamp);
574 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
575 const unsigned long *mask, bool timestamp)
578 int length, i, largest = 0;
580 /* How much space will the demuxed element take? */
581 for_each_set_bit(i, mask,
582 indio_dev->masklength) {
583 length = iio_storage_bytes_for_si(indio_dev, i);
584 bytes = ALIGN(bytes, length);
586 largest = max(largest, length);
590 length = iio_storage_bytes_for_timestamp(indio_dev);
591 bytes = ALIGN(bytes, length);
593 largest = max(largest, length);
596 bytes = ALIGN(bytes, largest);
600 static void iio_buffer_activate(struct iio_dev *indio_dev,
601 struct iio_buffer *buffer)
603 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
605 iio_buffer_get(buffer);
606 list_add(&buffer->buffer_list, &iio_dev_opaque->buffer_list);
609 static void iio_buffer_deactivate(struct iio_buffer *buffer)
611 list_del_init(&buffer->buffer_list);
612 wake_up_interruptible(&buffer->pollq);
613 iio_buffer_put(buffer);
616 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)
618 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
619 struct iio_buffer *buffer, *_buffer;
621 list_for_each_entry_safe(buffer, _buffer,
622 &iio_dev_opaque->buffer_list, buffer_list)
623 iio_buffer_deactivate(buffer);
626 static int iio_buffer_enable(struct iio_buffer *buffer,
627 struct iio_dev *indio_dev)
629 if (!buffer->access->enable)
631 return buffer->access->enable(buffer, indio_dev);
634 static int iio_buffer_disable(struct iio_buffer *buffer,
635 struct iio_dev *indio_dev)
637 if (!buffer->access->disable)
639 return buffer->access->disable(buffer, indio_dev);
642 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
643 struct iio_buffer *buffer)
647 if (!buffer->access->set_bytes_per_datum)
650 bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,
651 buffer->scan_timestamp);
653 buffer->access->set_bytes_per_datum(buffer, bytes);
656 static int iio_buffer_request_update(struct iio_dev *indio_dev,
657 struct iio_buffer *buffer)
661 iio_buffer_update_bytes_per_datum(indio_dev, buffer);
662 if (buffer->access->request_update) {
663 ret = buffer->access->request_update(buffer);
665 dev_dbg(&indio_dev->dev,
666 "Buffer not started: buffer parameter update failed (%d)\n",
675 static void iio_free_scan_mask(struct iio_dev *indio_dev,
676 const unsigned long *mask)
678 /* If the mask is dynamically allocated free it, otherwise do nothing */
679 if (!indio_dev->available_scan_masks)
683 struct iio_device_config {
685 unsigned int watermark;
686 const unsigned long *scan_mask;
687 unsigned int scan_bytes;
691 static int iio_verify_update(struct iio_dev *indio_dev,
692 struct iio_buffer *insert_buffer, struct iio_buffer *remove_buffer,
693 struct iio_device_config *config)
695 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
696 unsigned long *compound_mask;
697 const unsigned long *scan_mask;
698 bool strict_scanmask = false;
699 struct iio_buffer *buffer;
704 bitmap_empty(insert_buffer->scan_mask, indio_dev->masklength)) {
705 dev_dbg(&indio_dev->dev,
706 "At least one scan element must be enabled first\n");
710 memset(config, 0, sizeof(*config));
711 config->watermark = ~0;
714 * If there is just one buffer and we are removing it there is nothing
717 if (remove_buffer && !insert_buffer &&
718 list_is_singular(&iio_dev_opaque->buffer_list))
721 modes = indio_dev->modes;
723 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
724 if (buffer == remove_buffer)
726 modes &= buffer->access->modes;
727 config->watermark = min(config->watermark, buffer->watermark);
731 modes &= insert_buffer->access->modes;
732 config->watermark = min(config->watermark,
733 insert_buffer->watermark);
736 /* Definitely possible for devices to support both of these. */
737 if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {
738 config->mode = INDIO_BUFFER_TRIGGERED;
739 } else if (modes & INDIO_BUFFER_HARDWARE) {
741 * Keep things simple for now and only allow a single buffer to
742 * be connected in hardware mode.
744 if (insert_buffer && !list_empty(&iio_dev_opaque->buffer_list))
746 config->mode = INDIO_BUFFER_HARDWARE;
747 strict_scanmask = true;
748 } else if (modes & INDIO_BUFFER_SOFTWARE) {
749 config->mode = INDIO_BUFFER_SOFTWARE;
751 /* Can only occur on first buffer */
752 if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)
753 dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");
757 /* What scan mask do we actually have? */
758 compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);
759 if (compound_mask == NULL)
762 scan_timestamp = false;
764 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
765 if (buffer == remove_buffer)
767 bitmap_or(compound_mask, compound_mask, buffer->scan_mask,
768 indio_dev->masklength);
769 scan_timestamp |= buffer->scan_timestamp;
773 bitmap_or(compound_mask, compound_mask,
774 insert_buffer->scan_mask, indio_dev->masklength);
775 scan_timestamp |= insert_buffer->scan_timestamp;
778 if (indio_dev->available_scan_masks) {
779 scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,
780 indio_dev->masklength,
783 bitmap_free(compound_mask);
784 if (scan_mask == NULL)
787 scan_mask = compound_mask;
790 config->scan_bytes = iio_compute_scan_bytes(indio_dev,
791 scan_mask, scan_timestamp);
792 config->scan_mask = scan_mask;
793 config->scan_timestamp = scan_timestamp;
799 * struct iio_demux_table - table describing demux memcpy ops
800 * @from: index to copy from
801 * @to: index to copy to
802 * @length: how many bytes to copy
803 * @l: list head used for management
805 struct iio_demux_table {
812 static void iio_buffer_demux_free(struct iio_buffer *buffer)
814 struct iio_demux_table *p, *q;
815 list_for_each_entry_safe(p, q, &buffer->demux_list, l) {
821 static int iio_buffer_add_demux(struct iio_buffer *buffer,
822 struct iio_demux_table **p, unsigned int in_loc, unsigned int out_loc,
826 if (*p && (*p)->from + (*p)->length == in_loc &&
827 (*p)->to + (*p)->length == out_loc) {
828 (*p)->length += length;
830 *p = kmalloc(sizeof(**p), GFP_KERNEL);
835 (*p)->length = length;
836 list_add_tail(&(*p)->l, &buffer->demux_list);
842 static int iio_buffer_update_demux(struct iio_dev *indio_dev,
843 struct iio_buffer *buffer)
845 int ret, in_ind = -1, out_ind, length;
846 unsigned in_loc = 0, out_loc = 0;
847 struct iio_demux_table *p = NULL;
849 /* Clear out any old demux */
850 iio_buffer_demux_free(buffer);
851 kfree(buffer->demux_bounce);
852 buffer->demux_bounce = NULL;
854 /* First work out which scan mode we will actually have */
855 if (bitmap_equal(indio_dev->active_scan_mask,
857 indio_dev->masklength))
860 /* Now we have the two masks, work from least sig and build up sizes */
861 for_each_set_bit(out_ind,
863 indio_dev->masklength) {
864 in_ind = find_next_bit(indio_dev->active_scan_mask,
865 indio_dev->masklength,
867 while (in_ind != out_ind) {
868 length = iio_storage_bytes_for_si(indio_dev, in_ind);
869 /* Make sure we are aligned */
870 in_loc = roundup(in_loc, length) + length;
871 in_ind = find_next_bit(indio_dev->active_scan_mask,
872 indio_dev->masklength,
875 length = iio_storage_bytes_for_si(indio_dev, in_ind);
876 out_loc = roundup(out_loc, length);
877 in_loc = roundup(in_loc, length);
878 ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
880 goto error_clear_mux_table;
884 /* Relies on scan_timestamp being last */
885 if (buffer->scan_timestamp) {
886 length = iio_storage_bytes_for_timestamp(indio_dev);
887 out_loc = roundup(out_loc, length);
888 in_loc = roundup(in_loc, length);
889 ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
891 goto error_clear_mux_table;
895 buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);
896 if (buffer->demux_bounce == NULL) {
898 goto error_clear_mux_table;
902 error_clear_mux_table:
903 iio_buffer_demux_free(buffer);
908 static int iio_update_demux(struct iio_dev *indio_dev)
910 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
911 struct iio_buffer *buffer;
914 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
915 ret = iio_buffer_update_demux(indio_dev, buffer);
917 goto error_clear_mux_table;
921 error_clear_mux_table:
922 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list)
923 iio_buffer_demux_free(buffer);
928 static int iio_enable_buffers(struct iio_dev *indio_dev,
929 struct iio_device_config *config)
931 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
932 struct iio_buffer *buffer;
935 indio_dev->active_scan_mask = config->scan_mask;
936 indio_dev->scan_timestamp = config->scan_timestamp;
937 indio_dev->scan_bytes = config->scan_bytes;
938 indio_dev->currentmode = config->mode;
940 iio_update_demux(indio_dev);
943 if (indio_dev->setup_ops->preenable) {
944 ret = indio_dev->setup_ops->preenable(indio_dev);
946 dev_dbg(&indio_dev->dev,
947 "Buffer not started: buffer preenable failed (%d)\n", ret);
948 goto err_undo_config;
952 if (indio_dev->info->update_scan_mode) {
953 ret = indio_dev->info
954 ->update_scan_mode(indio_dev,
955 indio_dev->active_scan_mask);
957 dev_dbg(&indio_dev->dev,
958 "Buffer not started: update scan mode failed (%d)\n",
960 goto err_run_postdisable;
964 if (indio_dev->info->hwfifo_set_watermark)
965 indio_dev->info->hwfifo_set_watermark(indio_dev,
968 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
969 ret = iio_buffer_enable(buffer, indio_dev);
971 goto err_disable_buffers;
974 if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {
975 ret = iio_trigger_attach_poll_func(indio_dev->trig,
976 indio_dev->pollfunc);
978 goto err_disable_buffers;
981 if (indio_dev->setup_ops->postenable) {
982 ret = indio_dev->setup_ops->postenable(indio_dev);
984 dev_dbg(&indio_dev->dev,
985 "Buffer not started: postenable failed (%d)\n", ret);
986 goto err_detach_pollfunc;
993 if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {
994 iio_trigger_detach_poll_func(indio_dev->trig,
995 indio_dev->pollfunc);
998 list_for_each_entry_continue_reverse(buffer, &iio_dev_opaque->buffer_list,
1000 iio_buffer_disable(buffer, indio_dev);
1001 err_run_postdisable:
1002 if (indio_dev->setup_ops->postdisable)
1003 indio_dev->setup_ops->postdisable(indio_dev);
1005 indio_dev->currentmode = INDIO_DIRECT_MODE;
1006 indio_dev->active_scan_mask = NULL;
1011 static int iio_disable_buffers(struct iio_dev *indio_dev)
1013 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1014 struct iio_buffer *buffer;
1018 /* Wind down existing buffers - iff there are any */
1019 if (list_empty(&iio_dev_opaque->buffer_list))
1023 * If things go wrong at some step in disable we still need to continue
1024 * to perform the other steps, otherwise we leave the device in a
1025 * inconsistent state. We return the error code for the first error we
1029 if (indio_dev->setup_ops->predisable) {
1030 ret2 = indio_dev->setup_ops->predisable(indio_dev);
1035 if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {
1036 iio_trigger_detach_poll_func(indio_dev->trig,
1037 indio_dev->pollfunc);
1040 list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
1041 ret2 = iio_buffer_disable(buffer, indio_dev);
1046 if (indio_dev->setup_ops->postdisable) {
1047 ret2 = indio_dev->setup_ops->postdisable(indio_dev);
1052 iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);
1053 indio_dev->active_scan_mask = NULL;
1054 indio_dev->currentmode = INDIO_DIRECT_MODE;
1059 static int __iio_update_buffers(struct iio_dev *indio_dev,
1060 struct iio_buffer *insert_buffer,
1061 struct iio_buffer *remove_buffer)
1063 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1064 struct iio_device_config new_config;
1067 ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,
1072 if (insert_buffer) {
1073 ret = iio_buffer_request_update(indio_dev, insert_buffer);
1075 goto err_free_config;
1078 ret = iio_disable_buffers(indio_dev);
1080 goto err_deactivate_all;
1083 iio_buffer_deactivate(remove_buffer);
1085 iio_buffer_activate(indio_dev, insert_buffer);
1087 /* If no buffers in list, we are done */
1088 if (list_empty(&iio_dev_opaque->buffer_list))
1091 ret = iio_enable_buffers(indio_dev, &new_config);
1093 goto err_deactivate_all;
1099 * We've already verified that the config is valid earlier. If things go
1100 * wrong in either enable or disable the most likely reason is an IO
1101 * error from the device. In this case there is no good recovery
1102 * strategy. Just make sure to disable everything and leave the device
1103 * in a sane state. With a bit of luck the device might come back to
1104 * life again later and userspace can try again.
1106 iio_buffer_deactivate_all(indio_dev);
1109 iio_free_scan_mask(indio_dev, new_config.scan_mask);
1113 int iio_update_buffers(struct iio_dev *indio_dev,
1114 struct iio_buffer *insert_buffer,
1115 struct iio_buffer *remove_buffer)
1119 if (insert_buffer == remove_buffer)
1122 mutex_lock(&indio_dev->info_exist_lock);
1123 mutex_lock(&indio_dev->mlock);
1125 if (insert_buffer && iio_buffer_is_active(insert_buffer))
1126 insert_buffer = NULL;
1128 if (remove_buffer && !iio_buffer_is_active(remove_buffer))
1129 remove_buffer = NULL;
1131 if (!insert_buffer && !remove_buffer) {
1136 if (indio_dev->info == NULL) {
1141 ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);
1144 mutex_unlock(&indio_dev->mlock);
1145 mutex_unlock(&indio_dev->info_exist_lock);
1149 EXPORT_SYMBOL_GPL(iio_update_buffers);
1151 void iio_disable_all_buffers(struct iio_dev *indio_dev)
1153 iio_disable_buffers(indio_dev);
1154 iio_buffer_deactivate_all(indio_dev);
1157 static ssize_t iio_buffer_store_enable(struct device *dev,
1158 struct device_attribute *attr,
1163 bool requested_state;
1164 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1165 struct iio_buffer *buffer = indio_dev->buffer;
1168 ret = strtobool(buf, &requested_state);
1172 mutex_lock(&indio_dev->mlock);
1174 /* Find out if it is in the list */
1175 inlist = iio_buffer_is_active(buffer);
1176 /* Already in desired state */
1177 if (inlist == requested_state)
1180 if (requested_state)
1181 ret = __iio_update_buffers(indio_dev, buffer, NULL);
1183 ret = __iio_update_buffers(indio_dev, NULL, buffer);
1186 mutex_unlock(&indio_dev->mlock);
1187 return (ret < 0) ? ret : len;
1190 static const char * const iio_scan_elements_group_name = "scan_elements";
1192 static ssize_t iio_buffer_show_watermark(struct device *dev,
1193 struct device_attribute *attr,
1196 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1197 struct iio_buffer *buffer = indio_dev->buffer;
1199 return sprintf(buf, "%u\n", buffer->watermark);
1202 static ssize_t iio_buffer_store_watermark(struct device *dev,
1203 struct device_attribute *attr,
1207 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1208 struct iio_buffer *buffer = indio_dev->buffer;
1212 ret = kstrtouint(buf, 10, &val);
1218 mutex_lock(&indio_dev->mlock);
1220 if (val > buffer->length) {
1225 if (iio_buffer_is_active(buffer)) {
1230 buffer->watermark = val;
1232 mutex_unlock(&indio_dev->mlock);
1234 return ret ? ret : len;
1237 static ssize_t iio_dma_show_data_available(struct device *dev,
1238 struct device_attribute *attr,
1241 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1242 struct iio_buffer *buffer = indio_dev->buffer;
1244 return sprintf(buf, "%zu\n", iio_buffer_data_available(buffer));
1247 static DEVICE_ATTR(length, S_IRUGO | S_IWUSR, iio_buffer_read_length,
1248 iio_buffer_write_length);
1249 static struct device_attribute dev_attr_length_ro = __ATTR(length,
1250 S_IRUGO, iio_buffer_read_length, NULL);
1251 static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1252 iio_buffer_show_enable, iio_buffer_store_enable);
1253 static DEVICE_ATTR(watermark, S_IRUGO | S_IWUSR,
1254 iio_buffer_show_watermark, iio_buffer_store_watermark);
1255 static struct device_attribute dev_attr_watermark_ro = __ATTR(watermark,
1256 S_IRUGO, iio_buffer_show_watermark, NULL);
1257 static DEVICE_ATTR(data_available, S_IRUGO,
1258 iio_dma_show_data_available, NULL);
1260 static struct attribute *iio_buffer_attrs[] = {
1261 &dev_attr_length.attr,
1262 &dev_attr_enable.attr,
1263 &dev_attr_watermark.attr,
1264 &dev_attr_data_available.attr,
1267 static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,
1268 struct iio_dev *indio_dev)
1270 struct iio_dev_attr *p;
1271 struct attribute **attr;
1272 int ret, i, attrn, attrcount;
1273 const struct iio_chan_spec *channels;
1276 if (buffer->attrs) {
1277 while (buffer->attrs[attrcount] != NULL)
1281 attr = kcalloc(attrcount + ARRAY_SIZE(iio_buffer_attrs) + 1,
1282 sizeof(struct attribute *), GFP_KERNEL);
1286 memcpy(attr, iio_buffer_attrs, sizeof(iio_buffer_attrs));
1287 if (!buffer->access->set_length)
1288 attr[0] = &dev_attr_length_ro.attr;
1290 if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)
1291 attr[2] = &dev_attr_watermark_ro.attr;
1294 memcpy(&attr[ARRAY_SIZE(iio_buffer_attrs)], buffer->attrs,
1295 sizeof(struct attribute *) * attrcount);
1297 attr[attrcount + ARRAY_SIZE(iio_buffer_attrs)] = NULL;
1299 buffer->buffer_group.name = "buffer";
1300 buffer->buffer_group.attrs = attr;
1302 indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;
1305 INIT_LIST_HEAD(&buffer->scan_el_dev_attr_list);
1306 channels = indio_dev->channels;
1309 for (i = 0; i < indio_dev->num_channels; i++) {
1310 if (channels[i].scan_index < 0)
1313 ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,
1316 goto error_cleanup_dynamic;
1318 if (channels[i].type == IIO_TIMESTAMP)
1319 indio_dev->scan_index_timestamp =
1320 channels[i].scan_index;
1322 if (indio_dev->masklength && buffer->scan_mask == NULL) {
1323 buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,
1325 if (buffer->scan_mask == NULL) {
1327 goto error_cleanup_dynamic;
1332 buffer->scan_el_group.name = iio_scan_elements_group_name;
1334 buffer->scan_el_group.attrs = kcalloc(attrcount + 1,
1335 sizeof(buffer->scan_el_group.attrs[0]),
1337 if (buffer->scan_el_group.attrs == NULL) {
1339 goto error_free_scan_mask;
1343 list_for_each_entry(p, &buffer->scan_el_dev_attr_list, l)
1344 buffer->scan_el_group.attrs[attrn++] = &p->dev_attr.attr;
1345 indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;
1349 error_free_scan_mask:
1350 bitmap_free(buffer->scan_mask);
1351 error_cleanup_dynamic:
1352 iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
1353 kfree(buffer->buffer_group.attrs);
1358 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
1360 struct iio_buffer *buffer = indio_dev->buffer;
1361 const struct iio_chan_spec *channels;
1364 channels = indio_dev->channels;
1366 int ml = indio_dev->masklength;
1368 for (i = 0; i < indio_dev->num_channels; i++)
1369 ml = max(ml, channels[i].scan_index + 1);
1370 indio_dev->masklength = ml;
1376 return __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev);
1379 static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer)
1381 bitmap_free(buffer->scan_mask);
1382 kfree(buffer->buffer_group.attrs);
1383 kfree(buffer->scan_el_group.attrs);
1384 iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
1387 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)
1389 struct iio_buffer *buffer = indio_dev->buffer;
1394 __iio_buffer_free_sysfs_and_mask(buffer);
1398 * iio_validate_scan_mask_onehot() - Validates that exactly one channel is selected
1399 * @indio_dev: the iio device
1400 * @mask: scan mask to be checked
1402 * Return true if exactly one bit is set in the scan mask, false otherwise. It
1403 * can be used for devices where only one channel can be active for sampling at
1406 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,
1407 const unsigned long *mask)
1409 return bitmap_weight(mask, indio_dev->masklength) == 1;
1411 EXPORT_SYMBOL_GPL(iio_validate_scan_mask_onehot);
1413 static const void *iio_demux(struct iio_buffer *buffer,
1416 struct iio_demux_table *t;
1418 if (list_empty(&buffer->demux_list))
1420 list_for_each_entry(t, &buffer->demux_list, l)
1421 memcpy(buffer->demux_bounce + t->to,
1422 datain + t->from, t->length);
1424 return buffer->demux_bounce;
1427 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)
1429 const void *dataout = iio_demux(buffer, data);
1432 ret = buffer->access->store_to(buffer, dataout);
1437 * We can't just test for watermark to decide if we wake the poll queue
1438 * because read may request less samples than the watermark.
1440 wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);
1445 * iio_push_to_buffers() - push to a registered buffer.
1446 * @indio_dev: iio_dev structure for device.
1449 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)
1451 struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1453 struct iio_buffer *buf;
1455 list_for_each_entry(buf, &iio_dev_opaque->buffer_list, buffer_list) {
1456 ret = iio_push_to_buffer(buf, data);
1463 EXPORT_SYMBOL_GPL(iio_push_to_buffers);
1466 * iio_buffer_release() - Free a buffer's resources
1467 * @ref: Pointer to the kref embedded in the iio_buffer struct
1469 * This function is called when the last reference to the buffer has been
1470 * dropped. It will typically free all resources allocated by the buffer. Do not
1471 * call this function manually, always use iio_buffer_put() when done using a
1474 static void iio_buffer_release(struct kref *ref)
1476 struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);
1478 buffer->access->release(buffer);
1482 * iio_buffer_get() - Grab a reference to the buffer
1483 * @buffer: The buffer to grab a reference for, may be NULL
1485 * Returns the pointer to the buffer that was passed into the function.
1487 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)
1490 kref_get(&buffer->ref);
1494 EXPORT_SYMBOL_GPL(iio_buffer_get);
1497 * iio_buffer_put() - Release the reference to the buffer
1498 * @buffer: The buffer to release the reference for, may be NULL
1500 void iio_buffer_put(struct iio_buffer *buffer)
1503 kref_put(&buffer->ref, iio_buffer_release);
1505 EXPORT_SYMBOL_GPL(iio_buffer_put);
1508 * iio_device_attach_buffer - Attach a buffer to a IIO device
1509 * @indio_dev: The device the buffer should be attached to
1510 * @buffer: The buffer to attach to the device
1512 * This function attaches a buffer to a IIO device. The buffer stays attached to
1513 * the device until the device is freed. The function should only be called at
1514 * most once per device.
1516 void iio_device_attach_buffer(struct iio_dev *indio_dev,
1517 struct iio_buffer *buffer)
1519 indio_dev->buffer = iio_buffer_get(buffer);
1521 EXPORT_SYMBOL_GPL(iio_device_attach_buffer);