1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring.h>
7 #include <trace/events/io_uring.h>
9 #include <uapi/linux/io_uring.h>
20 struct list_head list;
21 /* head of the link, used by linked timeouts only */
22 struct io_kiocb *head;
23 /* for linked completions */
24 struct io_kiocb *prev;
27 struct io_timeout_rem {
37 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
39 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
44 static inline void io_put_req(struct io_kiocb *req)
46 if (req_ref_put_and_test(req)) {
52 static bool io_kill_timeout(struct io_kiocb *req, int status)
53 __must_hold(&req->ctx->completion_lock)
54 __must_hold(&req->ctx->timeout_lock)
56 struct io_timeout_data *io = req->async_data;
58 if (hrtimer_try_to_cancel(&io->timer) != -1) {
59 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
63 atomic_set(&req->ctx->cq_timeouts,
64 atomic_read(&req->ctx->cq_timeouts) + 1);
65 list_del_init(&timeout->list);
66 io_req_queue_tw_complete(req, status);
72 __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
73 __must_hold(&ctx->completion_lock)
76 struct io_timeout *timeout, *tmp;
78 spin_lock_irq(&ctx->timeout_lock);
79 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
81 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
82 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
83 u32 events_needed, events_got;
85 if (io_is_timeout_noseq(req))
89 * Since seq can easily wrap around over time, subtract
90 * the last seq at which timeouts were flushed before comparing.
91 * Assuming not more than 2^31-1 events have happened since,
92 * these subtractions won't have wrapped, so we can check if
93 * target is in [last_seq, current_seq] by comparing the two.
95 events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
96 events_got = seq - ctx->cq_last_tm_flush;
97 if (events_got < events_needed)
100 io_kill_timeout(req, 0);
102 ctx->cq_last_tm_flush = seq;
103 spin_unlock_irq(&ctx->timeout_lock);
106 static void io_req_tw_fail_links(struct io_kiocb *link, bool *locked)
108 io_tw_lock(link->ctx, locked);
110 struct io_kiocb *nxt = link->link;
111 long res = -ECANCELED;
113 if (link->flags & REQ_F_FAIL)
116 io_req_set_res(link, res, 0);
117 io_req_task_complete(link, locked);
122 static void io_fail_links(struct io_kiocb *req)
123 __must_hold(&req->ctx->completion_lock)
125 struct io_kiocb *link = req->link;
126 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
133 link->flags |= REQ_F_CQE_SKIP;
135 link->flags &= ~REQ_F_CQE_SKIP;
136 trace_io_uring_fail_link(req, link);
141 link->io_task_work.func = io_req_tw_fail_links;
142 io_req_task_work_add(link);
146 static inline void io_remove_next_linked(struct io_kiocb *req)
148 struct io_kiocb *nxt = req->link;
150 req->link = nxt->link;
154 void io_disarm_next(struct io_kiocb *req)
155 __must_hold(&req->ctx->completion_lock)
157 struct io_kiocb *link = NULL;
159 if (req->flags & REQ_F_ARM_LTIMEOUT) {
161 req->flags &= ~REQ_F_ARM_LTIMEOUT;
162 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
163 io_remove_next_linked(req);
164 io_req_queue_tw_complete(link, -ECANCELED);
166 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
167 struct io_ring_ctx *ctx = req->ctx;
169 spin_lock_irq(&ctx->timeout_lock);
170 link = io_disarm_linked_timeout(req);
171 spin_unlock_irq(&ctx->timeout_lock);
173 io_req_queue_tw_complete(link, -ECANCELED);
175 if (unlikely((req->flags & REQ_F_FAIL) &&
176 !(req->flags & REQ_F_HARDLINK)))
180 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
181 struct io_kiocb *link)
182 __must_hold(&req->ctx->completion_lock)
183 __must_hold(&req->ctx->timeout_lock)
185 struct io_timeout_data *io = link->async_data;
186 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout);
188 io_remove_next_linked(req);
189 timeout->head = NULL;
190 if (hrtimer_try_to_cancel(&io->timer) != -1) {
191 list_del(&timeout->list);
198 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
200 struct io_timeout_data *data = container_of(timer,
201 struct io_timeout_data, timer);
202 struct io_kiocb *req = data->req;
203 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
204 struct io_ring_ctx *ctx = req->ctx;
207 spin_lock_irqsave(&ctx->timeout_lock, flags);
208 list_del_init(&timeout->list);
209 atomic_set(&req->ctx->cq_timeouts,
210 atomic_read(&req->ctx->cq_timeouts) + 1);
211 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
213 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
216 io_req_set_res(req, -ETIME, 0);
217 req->io_task_work.func = io_req_task_complete;
218 io_req_task_work_add(req);
219 return HRTIMER_NORESTART;
222 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
223 struct io_cancel_data *cd)
224 __must_hold(&ctx->timeout_lock)
226 struct io_timeout *timeout;
227 struct io_timeout_data *io;
228 struct io_kiocb *req = NULL;
230 list_for_each_entry(timeout, &ctx->timeout_list, list) {
231 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
233 if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
234 cd->data != tmp->cqe.user_data)
236 if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
237 if (cd->seq == tmp->work.cancel_seq)
239 tmp->work.cancel_seq = cd->seq;
245 return ERR_PTR(-ENOENT);
247 io = req->async_data;
248 if (hrtimer_try_to_cancel(&io->timer) == -1)
249 return ERR_PTR(-EALREADY);
250 timeout = io_kiocb_to_cmd(req, struct io_timeout);
251 list_del_init(&timeout->list);
255 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
256 __must_hold(&ctx->completion_lock)
258 struct io_kiocb *req;
260 spin_lock_irq(&ctx->timeout_lock);
261 req = io_timeout_extract(ctx, cd);
262 spin_unlock_irq(&ctx->timeout_lock);
266 io_req_task_queue_fail(req, -ECANCELED);
270 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
272 unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED;
273 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
274 struct io_kiocb *prev = timeout->prev;
278 if (!(req->task->flags & PF_EXITING)) {
279 struct io_cancel_data cd = {
281 .data = prev->cqe.user_data,
284 ret = io_try_cancel(req->task->io_uring, &cd, issue_flags);
286 io_req_set_res(req, ret ?: -ETIME, 0);
287 io_req_task_complete(req, locked);
290 io_req_set_res(req, -ETIME, 0);
291 io_req_task_complete(req, locked);
295 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
297 struct io_timeout_data *data = container_of(timer,
298 struct io_timeout_data, timer);
299 struct io_kiocb *prev, *req = data->req;
300 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
301 struct io_ring_ctx *ctx = req->ctx;
304 spin_lock_irqsave(&ctx->timeout_lock, flags);
305 prev = timeout->head;
306 timeout->head = NULL;
309 * We don't expect the list to be empty, that will only happen if we
310 * race with the completion of the linked work.
313 io_remove_next_linked(prev);
314 if (!req_ref_inc_not_zero(prev))
317 list_del(&timeout->list);
318 timeout->prev = prev;
319 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
321 req->io_task_work.func = io_req_task_link_timeout;
322 io_req_task_work_add(req);
323 return HRTIMER_NORESTART;
326 static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
328 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
329 case IORING_TIMEOUT_BOOTTIME:
330 return CLOCK_BOOTTIME;
331 case IORING_TIMEOUT_REALTIME:
332 return CLOCK_REALTIME;
334 /* can't happen, vetted at prep time */
338 return CLOCK_MONOTONIC;
342 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
343 struct timespec64 *ts, enum hrtimer_mode mode)
344 __must_hold(&ctx->timeout_lock)
346 struct io_timeout_data *io;
347 struct io_timeout *timeout;
348 struct io_kiocb *req = NULL;
350 list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
351 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
353 if (user_data == tmp->cqe.user_data) {
361 io = req->async_data;
362 if (hrtimer_try_to_cancel(&io->timer) == -1)
364 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
365 io->timer.function = io_link_timeout_fn;
366 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
370 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
371 struct timespec64 *ts, enum hrtimer_mode mode)
372 __must_hold(&ctx->timeout_lock)
374 struct io_cancel_data cd = { .data = user_data, };
375 struct io_kiocb *req = io_timeout_extract(ctx, &cd);
376 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
377 struct io_timeout_data *data;
382 timeout->off = 0; /* noseq */
383 data = req->async_data;
384 list_add_tail(&timeout->list, &ctx->timeout_list);
385 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
386 data->timer.function = io_timeout_fn;
387 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
391 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
393 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
395 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
397 if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
400 tr->ltimeout = false;
401 tr->addr = READ_ONCE(sqe->addr);
402 tr->flags = READ_ONCE(sqe->timeout_flags);
403 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
404 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
406 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
408 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
410 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
412 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
414 } else if (tr->flags) {
415 /* timeout removal doesn't support flags */
422 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
424 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
429 * Remove or update an existing timeout command
431 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
433 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
434 struct io_ring_ctx *ctx = req->ctx;
437 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
438 struct io_cancel_data cd = { .data = tr->addr, };
440 spin_lock(&ctx->completion_lock);
441 ret = io_timeout_cancel(ctx, &cd);
442 spin_unlock(&ctx->completion_lock);
444 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
446 spin_lock_irq(&ctx->timeout_lock);
448 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
450 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
451 spin_unlock_irq(&ctx->timeout_lock);
456 io_req_set_res(req, ret, 0);
460 static int __io_timeout_prep(struct io_kiocb *req,
461 const struct io_uring_sqe *sqe,
462 bool is_timeout_link)
464 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
465 struct io_timeout_data *data;
467 u32 off = READ_ONCE(sqe->off);
469 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
471 if (off && is_timeout_link)
473 flags = READ_ONCE(sqe->timeout_flags);
474 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
475 IORING_TIMEOUT_ETIME_SUCCESS))
477 /* more than one clock specified is invalid, obviously */
478 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
481 INIT_LIST_HEAD(&timeout->list);
483 if (unlikely(off && !req->ctx->off_timeout_used))
484 req->ctx->off_timeout_used = true;
486 if (WARN_ON_ONCE(req_has_async_data(req)))
488 if (io_alloc_async_data(req))
491 data = req->async_data;
495 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
498 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
501 INIT_LIST_HEAD(&timeout->list);
502 data->mode = io_translate_timeout_mode(flags);
503 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
505 if (is_timeout_link) {
506 struct io_submit_link *link = &req->ctx->submit_state.link;
510 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
512 timeout->head = link->last;
513 link->last->flags |= REQ_F_ARM_LTIMEOUT;
518 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
520 return __io_timeout_prep(req, sqe, false);
523 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
525 return __io_timeout_prep(req, sqe, true);
528 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
530 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
531 struct io_ring_ctx *ctx = req->ctx;
532 struct io_timeout_data *data = req->async_data;
533 struct list_head *entry;
534 u32 tail, off = timeout->off;
536 spin_lock_irq(&ctx->timeout_lock);
539 * sqe->off holds how many events that need to occur for this
540 * timeout event to be satisfied. If it isn't set, then this is
541 * a pure timeout request, sequence isn't used.
543 if (io_is_timeout_noseq(req)) {
544 entry = ctx->timeout_list.prev;
548 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
549 timeout->target_seq = tail + off;
551 /* Update the last seq here in case io_flush_timeouts() hasn't.
552 * This is safe because ->completion_lock is held, and submissions
553 * and completions are never mixed in the same ->completion_lock section.
555 ctx->cq_last_tm_flush = tail;
558 * Insertion sort, ensuring the first entry in the list is always
559 * the one we need first.
561 list_for_each_prev(entry, &ctx->timeout_list) {
562 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
563 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
565 if (io_is_timeout_noseq(nxt))
567 /* nxt.seq is behind @tail, otherwise would've been completed */
568 if (off >= nextt->target_seq - tail)
572 list_add(&timeout->list, entry);
573 data->timer.function = io_timeout_fn;
574 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
575 spin_unlock_irq(&ctx->timeout_lock);
576 return IOU_ISSUE_SKIP_COMPLETE;
579 void io_queue_linked_timeout(struct io_kiocb *req)
581 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
582 struct io_ring_ctx *ctx = req->ctx;
584 spin_lock_irq(&ctx->timeout_lock);
586 * If the back reference is NULL, then our linked request finished
587 * before we got a chance to setup the timer
590 struct io_timeout_data *data = req->async_data;
592 data->timer.function = io_link_timeout_fn;
593 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
595 list_add_tail(&timeout->list, &ctx->ltimeout_list);
597 spin_unlock_irq(&ctx->timeout_lock);
598 /* drop submission reference */
602 static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
604 __must_hold(&req->ctx->timeout_lock)
606 struct io_kiocb *req;
608 if (task && head->task != task)
613 io_for_each_link(req, head) {
614 if (req->flags & REQ_F_INFLIGHT)
620 /* Returns true if we found and killed one or more timeouts */
621 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
624 struct io_timeout *timeout, *tmp;
628 spin_lock_irq(&ctx->timeout_lock);
629 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
630 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
632 if (io_match_task(req, tsk, cancel_all) &&
633 io_kill_timeout(req, -ECANCELED))
636 spin_unlock_irq(&ctx->timeout_lock);
637 io_cq_unlock_post(ctx);
638 return canceled != 0;