1 // SPDX-License-Identifier: GPL-2.0
5 #include <linux/freezer.h>
6 #include <linux/kthread.h>
7 #include <linux/preempt.h>
9 static inline long io_timer_cmp(io_timer_heap *h,
13 return l->expire - r->expire;
16 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
20 spin_lock(&clock->timer_lock);
22 if (time_after_eq((unsigned long) atomic64_read(&clock->now),
24 spin_unlock(&clock->timer_lock);
29 for (i = 0; i < clock->timers.used; i++)
30 if (clock->timers.data[i] == timer)
33 BUG_ON(!heap_add(&clock->timers, timer, io_timer_cmp, NULL));
35 spin_unlock(&clock->timer_lock);
38 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer)
42 spin_lock(&clock->timer_lock);
44 for (i = 0; i < clock->timers.used; i++)
45 if (clock->timers.data[i] == timer) {
46 heap_del(&clock->timers, i, io_timer_cmp, NULL);
50 spin_unlock(&clock->timer_lock);
53 struct io_clock_wait {
54 struct io_timer io_timer;
55 struct timer_list cpu_timer;
56 struct task_struct *task;
60 static void io_clock_wait_fn(struct io_timer *timer)
62 struct io_clock_wait *wait = container_of(timer,
63 struct io_clock_wait, io_timer);
66 wake_up_process(wait->task);
69 static void io_clock_cpu_timeout(struct timer_list *timer)
71 struct io_clock_wait *wait = container_of(timer,
72 struct io_clock_wait, cpu_timer);
75 wake_up_process(wait->task);
78 void bch2_io_clock_schedule_timeout(struct io_clock *clock, unsigned long until)
80 struct io_clock_wait wait;
82 /* XXX: calculate sleep time rigorously */
83 wait.io_timer.expire = until;
84 wait.io_timer.fn = io_clock_wait_fn;
87 bch2_io_timer_add(clock, &wait.io_timer);
91 bch2_io_timer_del(clock, &wait.io_timer);
94 void bch2_kthread_io_clock_wait(struct io_clock *clock,
95 unsigned long io_until,
96 unsigned long cpu_timeout)
98 bool kthread = (current->flags & PF_KTHREAD) != 0;
99 struct io_clock_wait wait;
101 wait.io_timer.expire = io_until;
102 wait.io_timer.fn = io_clock_wait_fn;
105 bch2_io_timer_add(clock, &wait.io_timer);
107 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0);
109 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT)
110 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies);
113 set_current_state(TASK_INTERRUPTIBLE);
114 if (kthread && kthread_should_stop())
124 __set_current_state(TASK_RUNNING);
125 del_timer_sync(&wait.cpu_timer);
126 destroy_timer_on_stack(&wait.cpu_timer);
127 bch2_io_timer_del(clock, &wait.io_timer);
130 static struct io_timer *get_expired_timer(struct io_clock *clock,
133 struct io_timer *ret = NULL;
135 spin_lock(&clock->timer_lock);
137 if (clock->timers.used &&
138 time_after_eq(now, clock->timers.data[0]->expire))
139 heap_pop(&clock->timers, ret, io_timer_cmp, NULL);
141 spin_unlock(&clock->timer_lock);
146 void __bch2_increment_clock(struct io_clock *clock, unsigned sectors)
148 struct io_timer *timer;
149 unsigned long now = atomic64_add_return(sectors, &clock->now);
151 while ((timer = get_expired_timer(clock, now)))
155 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock)
161 spin_lock(&clock->timer_lock);
162 now = atomic64_read(&clock->now);
164 for (i = 0; i < clock->timers.used; i++)
165 prt_printf(out, "%ps:\t%li\n",
166 clock->timers.data[i]->fn,
167 clock->timers.data[i]->expire - now);
168 spin_unlock(&clock->timer_lock);
172 void bch2_io_clock_exit(struct io_clock *clock)
174 free_heap(&clock->timers);
175 free_percpu(clock->pcpu_buf);
178 int bch2_io_clock_init(struct io_clock *clock)
180 atomic64_set(&clock->now, 0);
181 spin_lock_init(&clock->timer_lock);
183 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
185 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
186 if (!clock->pcpu_buf)
187 return -BCH_ERR_ENOMEM_io_clock_init;
189 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL))
190 return -BCH_ERR_ENOMEM_io_clock_init;