1 // SPDX-License-Identifier: GPL-2.0-only
3 * Tegra host1x Interrupt Management
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
9 #include <linux/interrupt.h>
10 #include <linux/slab.h>
11 #include <linux/irq.h>
13 #include <trace/events/host1x.h>
18 /* Wait list management */
27 static void waiter_release(struct kref *kref)
29 kfree(container_of(kref, struct host1x_waitlist, refcount));
33 * add a waiter to a waiter queue, sorted by threshold
34 * returns true if it was added at the head of the queue
36 static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
37 struct list_head *queue)
39 struct host1x_waitlist *pos;
40 u32 thresh = waiter->thresh;
42 list_for_each_entry_reverse(pos, queue, list)
43 if ((s32)(pos->thresh - thresh) <= 0) {
44 list_add(&waiter->list, &pos->list);
48 list_add(&waiter->list, queue);
53 * run through a waiter queue for a single sync point ID
54 * and gather all completed waiters into lists by actions
56 static void remove_completed_waiters(struct list_head *head, u32 sync,
57 struct list_head completed[HOST1X_INTR_ACTION_COUNT])
59 struct list_head *dest;
60 struct host1x_waitlist *waiter, *next, *prev;
62 list_for_each_entry_safe(waiter, next, head, list) {
63 if ((s32)(waiter->thresh - sync) > 0)
66 dest = completed + waiter->action;
68 /* consolidate submit cleanups */
69 if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
71 prev = list_entry(dest->prev,
72 struct host1x_waitlist, list);
73 if (prev->data == waiter->data) {
79 /* PENDING->REMOVED or CANCELLED->HANDLED */
80 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
81 list_del(&waiter->list);
82 kref_put(&waiter->refcount, waiter_release);
84 list_move_tail(&waiter->list, dest);
88 static void reset_threshold_interrupt(struct host1x *host,
89 struct list_head *head,
93 list_first_entry(head, struct host1x_waitlist, list)->thresh;
95 host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
96 host1x_hw_intr_enable_syncpt_intr(host, id);
99 static void action_submit_complete(struct host1x_waitlist *waiter)
101 struct host1x_channel *channel = waiter->data;
103 host1x_cdma_update(&channel->cdma);
105 /* Add nr_completed to trace */
106 trace_host1x_channel_submit_complete(dev_name(channel->dev),
107 waiter->count, waiter->thresh);
110 static void action_wakeup(struct host1x_waitlist *waiter)
112 wait_queue_head_t *wq = waiter->data;
117 static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
119 wait_queue_head_t *wq = waiter->data;
121 wake_up_interruptible(wq);
124 typedef void (*action_handler)(struct host1x_waitlist *waiter);
126 static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
127 action_submit_complete,
129 action_wakeup_interruptible,
132 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
134 struct list_head *head = completed;
137 for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
138 action_handler handler = action_handlers[i];
139 struct host1x_waitlist *waiter, *next;
141 list_for_each_entry_safe(waiter, next, head, list) {
142 list_del(&waiter->list);
144 WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
146 kref_put(&waiter->refcount, waiter_release);
152 * Remove & handle all waiters that have completed for the given syncpt
154 static int process_wait_list(struct host1x *host,
155 struct host1x_syncpt *syncpt,
158 struct list_head completed[HOST1X_INTR_ACTION_COUNT];
162 for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
163 INIT_LIST_HEAD(completed + i);
165 spin_lock(&syncpt->intr.lock);
167 remove_completed_waiters(&syncpt->intr.wait_head, threshold,
170 empty = list_empty(&syncpt->intr.wait_head);
172 host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
174 reset_threshold_interrupt(host, &syncpt->intr.wait_head,
177 spin_unlock(&syncpt->intr.lock);
179 run_handlers(completed);
185 * Sync point threshold interrupt service thread function
186 * Handles sync point threshold triggers, in thread context
189 static void syncpt_thresh_work(struct work_struct *work)
191 struct host1x_syncpt_intr *syncpt_intr =
192 container_of(work, struct host1x_syncpt_intr, work);
193 struct host1x_syncpt *syncpt =
194 container_of(syncpt_intr, struct host1x_syncpt, intr);
195 unsigned int id = syncpt->id;
196 struct host1x *host = syncpt->host;
198 (void)process_wait_list(host, syncpt,
199 host1x_syncpt_load(host->syncpt + id));
202 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
203 u32 thresh, enum host1x_intr_action action,
204 void *data, struct host1x_waitlist *waiter,
209 if (waiter == NULL) {
210 pr_warn("%s: NULL waiter\n", __func__);
214 /* initialize a new waiter */
215 INIT_LIST_HEAD(&waiter->list);
216 kref_init(&waiter->refcount);
218 kref_get(&waiter->refcount);
219 waiter->thresh = thresh;
220 waiter->action = action;
221 atomic_set(&waiter->state, WLS_PENDING);
225 spin_lock(&syncpt->intr.lock);
227 queue_was_empty = list_empty(&syncpt->intr.wait_head);
229 if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
230 /* added at head of list - new threshold value */
231 host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
233 /* added as first waiter - enable interrupt */
235 host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
238 spin_unlock(&syncpt->intr.lock);
245 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref)
247 struct host1x_waitlist *waiter = ref;
248 struct host1x_syncpt *syncpt;
250 while (atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED) ==
254 syncpt = host->syncpt + id;
255 (void)process_wait_list(host, syncpt,
256 host1x_syncpt_load(host->syncpt + id));
258 kref_put(&waiter->refcount, waiter_release);
261 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
264 u32 nb_pts = host1x_syncpt_nb_pts(host);
266 mutex_init(&host->intr_mutex);
267 host->intr_syncpt_irq = irq_sync;
269 for (id = 0; id < nb_pts; ++id) {
270 struct host1x_syncpt *syncpt = host->syncpt + id;
272 spin_lock_init(&syncpt->intr.lock);
273 INIT_LIST_HEAD(&syncpt->intr.wait_head);
274 snprintf(syncpt->intr.thresh_irq_name,
275 sizeof(syncpt->intr.thresh_irq_name),
276 "host1x_sp_%02u", id);
279 host1x_intr_start(host);
284 void host1x_intr_deinit(struct host1x *host)
286 host1x_intr_stop(host);
289 void host1x_intr_start(struct host1x *host)
291 u32 hz = clk_get_rate(host->clk);
294 mutex_lock(&host->intr_mutex);
295 err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
298 mutex_unlock(&host->intr_mutex);
301 mutex_unlock(&host->intr_mutex);
304 void host1x_intr_stop(struct host1x *host)
307 struct host1x_syncpt *syncpt = host->syncpt;
308 u32 nb_pts = host1x_syncpt_nb_pts(host);
310 mutex_lock(&host->intr_mutex);
312 host1x_hw_intr_disable_all_syncpt_intrs(host);
314 for (id = 0; id < nb_pts; ++id) {
315 struct host1x_waitlist *waiter, *next;
317 list_for_each_entry_safe(waiter, next,
318 &syncpt[id].intr.wait_head, list) {
319 if (atomic_cmpxchg(&waiter->state,
320 WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
321 list_del(&waiter->list);
322 kref_put(&waiter->refcount, waiter_release);
326 if (!list_empty(&syncpt[id].intr.wait_head)) {
327 /* output diagnostics */
328 mutex_unlock(&host->intr_mutex);
329 pr_warn("%s cannot stop syncpt intr id=%u\n",
335 host1x_hw_intr_free_syncpt_irq(host);
337 mutex_unlock(&host->intr_mutex);