1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
10 /* Implements a simple circular buffer that is shared between user
11 * and the driver and can be resized. The requested element size is
12 * rounded up to a power of 2 and the number of elements in the buffer
13 * is also rounded up to a power of 2. Since the queue is empty when
14 * the producer and consumer indices match the maximum capacity of the
15 * queue is one less than the number of element slots.
18 * - The driver indices are always masked off to q->index_mask
19 * before storing so do not need to be checked on reads.
20 * - The user whether user space or kernel is generally
21 * not trusted so its parameters are masked to make sure
22 * they do not access the queue out of bounds on reads.
23 * - The driver indices for queues must not be written
24 * by user so a local copy is used and a shared copy is
25 * stored when the local copy is changed.
26 * - By passing the type in the parameter list separate from q
27 * the compiler can eliminate the switch statement when the
28 * actual queue type is known when the function is called at
30 * - These queues are lock free. The user and driver must protect
31 * changes to their end of the queues with locks if more than one
32 * CPU can be accessing it at the same time.
36 * enum queue_type - type of queue
37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
38 * read by client which may be a user space
39 * application or a kernel ulp.
40 * Used by rxe internals only.
41 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
43 * Used by rxe internals only.
44 * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and
46 * Used by kernel verbs APIs only on
48 * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and
50 * Used by kernel verbs APIs only on
55 QUEUE_TYPE_FROM_CLIENT,
64 struct rxe_queue_buf *buf;
65 struct rxe_mmap_info *ip;
68 unsigned int log2_elem_size;
71 /* private copy of index for shared queues between
72 * driver and clients. Driver reads and writes
73 * this copy and then replicates to rxe_queue_buf
74 * for read access by clients.
79 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
80 struct ib_udata *udata, struct rxe_queue_buf *buf,
81 size_t buf_size, struct rxe_mmap_info **ip_p);
83 void rxe_queue_reset(struct rxe_queue *q);
85 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
86 unsigned int elem_size, enum queue_type type);
88 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
89 unsigned int elem_size, struct ib_udata *udata,
90 struct mminfo __user *outbuf,
91 spinlock_t *producer_lock, spinlock_t *consumer_lock);
93 void rxe_queue_cleanup(struct rxe_queue *queue);
95 static inline u32 queue_next_index(struct rxe_queue *q, int index)
97 return (index + 1) & q->index_mask;
100 static inline u32 queue_get_producer(const struct rxe_queue *q,
101 enum queue_type type)
106 case QUEUE_TYPE_FROM_CLIENT:
107 /* used by rxe, client owns the index */
108 prod = smp_load_acquire(&q->buf->producer_index);
110 case QUEUE_TYPE_TO_CLIENT:
111 /* used by rxe which owns the index */
114 case QUEUE_TYPE_FROM_ULP:
115 /* used by ulp which owns the index */
116 prod = q->buf->producer_index;
118 case QUEUE_TYPE_TO_ULP:
119 /* used by ulp, rxe owns the index */
120 prod = smp_load_acquire(&q->buf->producer_index);
127 static inline u32 queue_get_consumer(const struct rxe_queue *q,
128 enum queue_type type)
133 case QUEUE_TYPE_FROM_CLIENT:
134 /* used by rxe which owns the index */
137 case QUEUE_TYPE_TO_CLIENT:
138 /* used by rxe, client owns the index */
139 cons = smp_load_acquire(&q->buf->consumer_index);
141 case QUEUE_TYPE_FROM_ULP:
142 /* used by ulp, rxe owns the index */
143 cons = smp_load_acquire(&q->buf->consumer_index);
145 case QUEUE_TYPE_TO_ULP:
146 /* used by ulp which owns the index */
147 cons = q->buf->consumer_index;
154 static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
156 u32 prod = queue_get_producer(q, type);
157 u32 cons = queue_get_consumer(q, type);
159 return ((prod - cons) & q->index_mask) == 0;
162 static inline int queue_full(struct rxe_queue *q, enum queue_type type)
164 u32 prod = queue_get_producer(q, type);
165 u32 cons = queue_get_consumer(q, type);
167 return ((prod + 1 - cons) & q->index_mask) == 0;
170 static inline u32 queue_count(const struct rxe_queue *q,
171 enum queue_type type)
173 u32 prod = queue_get_producer(q, type);
174 u32 cons = queue_get_consumer(q, type);
176 return (prod - cons) & q->index_mask;
179 static inline void queue_advance_producer(struct rxe_queue *q,
180 enum queue_type type)
185 case QUEUE_TYPE_FROM_CLIENT:
186 /* used by rxe, client owns the index */
188 pr_warn("%s: attempt to advance client index\n",
191 case QUEUE_TYPE_TO_CLIENT:
192 /* used by rxe which owns the index */
194 prod = (prod + 1) & q->index_mask;
196 /* release so client can read it safely */
197 smp_store_release(&q->buf->producer_index, prod);
199 case QUEUE_TYPE_FROM_ULP:
200 /* used by ulp which owns the index */
201 prod = q->buf->producer_index;
202 prod = (prod + 1) & q->index_mask;
203 /* release so rxe can read it safely */
204 smp_store_release(&q->buf->producer_index, prod);
206 case QUEUE_TYPE_TO_ULP:
207 /* used by ulp, rxe owns the index */
209 pr_warn("%s: attempt to advance driver index\n",
215 static inline void queue_advance_consumer(struct rxe_queue *q,
216 enum queue_type type)
221 case QUEUE_TYPE_FROM_CLIENT:
222 /* used by rxe which owns the index */
223 cons = (q->index + 1) & q->index_mask;
225 /* release so client can read it safely */
226 smp_store_release(&q->buf->consumer_index, cons);
228 case QUEUE_TYPE_TO_CLIENT:
229 /* used by rxe, client owns the index */
231 pr_warn("%s: attempt to advance client index\n",
234 case QUEUE_TYPE_FROM_ULP:
235 /* used by ulp, rxe owns the index */
237 pr_warn("%s: attempt to advance driver index\n",
240 case QUEUE_TYPE_TO_ULP:
241 /* used by ulp which owns the index */
242 cons = q->buf->consumer_index;
243 cons = (cons + 1) & q->index_mask;
244 /* release so rxe can read it safely */
245 smp_store_release(&q->buf->consumer_index, cons);
250 static inline void *queue_producer_addr(struct rxe_queue *q,
251 enum queue_type type)
253 u32 prod = queue_get_producer(q, type);
255 return q->buf->data + (prod << q->log2_elem_size);
258 static inline void *queue_consumer_addr(struct rxe_queue *q,
259 enum queue_type type)
261 u32 cons = queue_get_consumer(q, type);
263 return q->buf->data + (cons << q->log2_elem_size);
266 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
268 return q->buf->data + ((index & q->index_mask)
269 << q->log2_elem_size);
272 static inline u32 queue_index_from_addr(const struct rxe_queue *q,
275 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
279 static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
281 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
284 #endif /* RXE_QUEUE_H */