1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 * Copyright (C) 2005-2006 Thomas Gleixner
6 * This file contains driver APIs to the irq subsystem.
9 #define pr_fmt(fmt) "genirq: " fmt
11 #include <linux/irq.h>
12 #include <linux/kthread.h>
13 #include <linux/module.h>
14 #include <linux/random.h>
15 #include <linux/interrupt.h>
16 #include <linux/irqdomain.h>
17 #include <linux/slab.h>
18 #include <linux/sched.h>
19 #include <linux/sched/rt.h>
20 #include <linux/sched/task.h>
21 #include <uapi/linux/sched/types.h>
22 #include <linux/task_work.h>
24 #include "internals.h"
26 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
27 __read_mostly bool force_irqthreads;
28 EXPORT_SYMBOL_GPL(force_irqthreads);
30 static int __init setup_forced_irqthreads(char *arg)
32 force_irqthreads = true;
35 early_param("threadirqs", setup_forced_irqthreads);
38 static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
40 struct irq_data *irqd = irq_desc_get_irq_data(desc);
47 * Wait until we're out of the critical section. This might
48 * give the wrong answer due to the lack of memory barriers.
50 while (irqd_irq_inprogress(&desc->irq_data))
53 /* Ok, that indicated we're done: double-check carefully. */
54 raw_spin_lock_irqsave(&desc->lock, flags);
55 inprogress = irqd_irq_inprogress(&desc->irq_data);
58 * If requested and supported, check at the chip whether it
59 * is in flight at the hardware level, i.e. already pending
60 * in a CPU and waiting for service and acknowledge.
62 if (!inprogress && sync_chip) {
64 * Ignore the return code. inprogress is only updated
65 * when the chip supports it.
67 __irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,
70 raw_spin_unlock_irqrestore(&desc->lock, flags);
72 /* Oops, that failed? */
77 * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
78 * @irq: interrupt number to wait for
80 * This function waits for any pending hard IRQ handlers for this
81 * interrupt to complete before returning. If you use this
82 * function while holding a resource the IRQ handler may need you
83 * will deadlock. It does not take associated threaded handlers
86 * Do not use this for shutdown scenarios where you must be sure
87 * that all parts (hardirq and threaded handler) have completed.
89 * Returns: false if a threaded handler is active.
91 * This function may be called - with care - from IRQ context.
93 * It does not check whether there is an interrupt in flight at the
94 * hardware level, but not serviced yet, as this might deadlock when
95 * called with interrupts disabled and the target CPU of the interrupt
98 bool synchronize_hardirq(unsigned int irq)
100 struct irq_desc *desc = irq_to_desc(irq);
103 __synchronize_hardirq(desc, false);
104 return !atomic_read(&desc->threads_active);
109 EXPORT_SYMBOL(synchronize_hardirq);
112 * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
113 * @irq: interrupt number to wait for
115 * This function waits for any pending IRQ handlers for this interrupt
116 * to complete before returning. If you use this function while
117 * holding a resource the IRQ handler may need you will deadlock.
119 * Can only be called from preemptible code as it might sleep when
120 * an interrupt thread is associated to @irq.
122 * It optionally makes sure (when the irq chip supports that method)
123 * that the interrupt is not pending in any CPU and waiting for
126 void synchronize_irq(unsigned int irq)
128 struct irq_desc *desc = irq_to_desc(irq);
131 __synchronize_hardirq(desc, true);
133 * We made sure that no hardirq handler is
134 * running. Now verify that no threaded handlers are
137 wait_event(desc->wait_for_threads,
138 !atomic_read(&desc->threads_active));
141 EXPORT_SYMBOL(synchronize_irq);
144 cpumask_var_t irq_default_affinity;
146 static bool __irq_can_set_affinity(struct irq_desc *desc)
148 if (!desc || !irqd_can_balance(&desc->irq_data) ||
149 !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
155 * irq_can_set_affinity - Check if the affinity of a given irq can be set
156 * @irq: Interrupt to check
159 int irq_can_set_affinity(unsigned int irq)
161 return __irq_can_set_affinity(irq_to_desc(irq));
165 * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
166 * @irq: Interrupt to check
168 * Like irq_can_set_affinity() above, but additionally checks for the
169 * AFFINITY_MANAGED flag.
171 bool irq_can_set_affinity_usr(unsigned int irq)
173 struct irq_desc *desc = irq_to_desc(irq);
175 return __irq_can_set_affinity(desc) &&
176 !irqd_affinity_is_managed(&desc->irq_data);
180 * irq_set_thread_affinity - Notify irq threads to adjust affinity
181 * @desc: irq descriptor which has affitnity changed
183 * We just set IRQTF_AFFINITY and delegate the affinity setting
184 * to the interrupt thread itself. We can not call
185 * set_cpus_allowed_ptr() here as we hold desc->lock and this
186 * code can be called from hard interrupt context.
188 void irq_set_thread_affinity(struct irq_desc *desc)
190 struct irqaction *action;
192 for_each_action_of_desc(desc, action)
194 set_bit(IRQTF_AFFINITY, &action->thread_flags);
197 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
198 static void irq_validate_effective_affinity(struct irq_data *data)
200 const struct cpumask *m = irq_data_get_effective_affinity_mask(data);
201 struct irq_chip *chip = irq_data_get_irq_chip(data);
203 if (!cpumask_empty(m))
205 pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",
206 chip->name, data->irq);
209 static inline void irq_init_effective_affinity(struct irq_data *data,
210 const struct cpumask *mask)
212 cpumask_copy(irq_data_get_effective_affinity_mask(data), mask);
215 static inline void irq_validate_effective_affinity(struct irq_data *data) { }
216 static inline void irq_init_effective_affinity(struct irq_data *data,
217 const struct cpumask *mask) { }
220 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
223 struct irq_desc *desc = irq_data_to_desc(data);
224 struct irq_chip *chip = irq_data_get_irq_chip(data);
227 if (!chip || !chip->irq_set_affinity)
230 ret = chip->irq_set_affinity(data, mask, force);
232 case IRQ_SET_MASK_OK:
233 case IRQ_SET_MASK_OK_DONE:
234 cpumask_copy(desc->irq_common_data.affinity, mask);
236 case IRQ_SET_MASK_OK_NOCOPY:
237 irq_validate_effective_affinity(data);
238 irq_set_thread_affinity(desc);
245 #ifdef CONFIG_GENERIC_PENDING_IRQ
246 static inline int irq_set_affinity_pending(struct irq_data *data,
247 const struct cpumask *dest)
249 struct irq_desc *desc = irq_data_to_desc(data);
251 irqd_set_move_pending(data);
252 irq_copy_pending(desc, dest);
256 static inline int irq_set_affinity_pending(struct irq_data *data,
257 const struct cpumask *dest)
263 static int irq_try_set_affinity(struct irq_data *data,
264 const struct cpumask *dest, bool force)
266 int ret = irq_do_set_affinity(data, dest, force);
269 * In case that the underlying vector management is busy and the
270 * architecture supports the generic pending mechanism then utilize
271 * this to avoid returning an error to user space.
273 if (ret == -EBUSY && !force)
274 ret = irq_set_affinity_pending(data, dest);
278 static bool irq_set_affinity_deactivated(struct irq_data *data,
279 const struct cpumask *mask, bool force)
281 struct irq_desc *desc = irq_data_to_desc(data);
284 * Handle irq chips which can handle affinity only in activated
287 * If the interrupt is not yet activated, just store the affinity
288 * mask and do not call the chip driver at all. On activation the
289 * driver has to make sure anyway that the interrupt is in a
290 * useable state so startup works.
292 if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||
293 irqd_is_activated(data) || !irqd_affinity_on_activate(data))
296 cpumask_copy(desc->irq_common_data.affinity, mask);
297 irq_init_effective_affinity(data, mask);
298 irqd_set(data, IRQD_AFFINITY_SET);
302 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
305 struct irq_chip *chip = irq_data_get_irq_chip(data);
306 struct irq_desc *desc = irq_data_to_desc(data);
309 if (!chip || !chip->irq_set_affinity)
312 if (irq_set_affinity_deactivated(data, mask, force))
315 if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {
316 ret = irq_try_set_affinity(data, mask, force);
318 irqd_set_move_pending(data);
319 irq_copy_pending(desc, mask);
322 if (desc->affinity_notify) {
323 kref_get(&desc->affinity_notify->kref);
324 if (!schedule_work(&desc->affinity_notify->work)) {
325 /* Work was already scheduled, drop our extra ref */
326 kref_put(&desc->affinity_notify->kref,
327 desc->affinity_notify->release);
330 irqd_set(data, IRQD_AFFINITY_SET);
335 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
337 struct irq_desc *desc = irq_to_desc(irq);
344 raw_spin_lock_irqsave(&desc->lock, flags);
345 ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
346 raw_spin_unlock_irqrestore(&desc->lock, flags);
350 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
353 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
357 desc->affinity_hint = m;
358 irq_put_desc_unlock(desc, flags);
359 /* set the initial affinity to prevent every interrupt being on CPU0 */
361 __irq_set_affinity(irq, m, false);
364 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
366 static void irq_affinity_notify(struct work_struct *work)
368 struct irq_affinity_notify *notify =
369 container_of(work, struct irq_affinity_notify, work);
370 struct irq_desc *desc = irq_to_desc(notify->irq);
371 cpumask_var_t cpumask;
374 if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
377 raw_spin_lock_irqsave(&desc->lock, flags);
378 if (irq_move_pending(&desc->irq_data))
379 irq_get_pending(cpumask, desc);
381 cpumask_copy(cpumask, desc->irq_common_data.affinity);
382 raw_spin_unlock_irqrestore(&desc->lock, flags);
384 notify->notify(notify, cpumask);
386 free_cpumask_var(cpumask);
388 kref_put(¬ify->kref, notify->release);
392 * irq_set_affinity_notifier - control notification of IRQ affinity changes
393 * @irq: Interrupt for which to enable/disable notification
394 * @notify: Context for notification, or %NULL to disable
395 * notification. Function pointers must be initialised;
396 * the other fields will be initialised by this function.
398 * Must be called in process context. Notification may only be enabled
399 * after the IRQ is allocated and must be disabled before the IRQ is
400 * freed using free_irq().
403 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
405 struct irq_desc *desc = irq_to_desc(irq);
406 struct irq_affinity_notify *old_notify;
409 /* The release function is promised process context */
412 if (!desc || desc->istate & IRQS_NMI)
415 /* Complete initialisation of *notify */
418 kref_init(¬ify->kref);
419 INIT_WORK(¬ify->work, irq_affinity_notify);
422 raw_spin_lock_irqsave(&desc->lock, flags);
423 old_notify = desc->affinity_notify;
424 desc->affinity_notify = notify;
425 raw_spin_unlock_irqrestore(&desc->lock, flags);
428 if (cancel_work_sync(&old_notify->work)) {
429 /* Pending work had a ref, put that one too */
430 kref_put(&old_notify->kref, old_notify->release);
432 kref_put(&old_notify->kref, old_notify->release);
437 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
439 #ifndef CONFIG_AUTO_IRQ_AFFINITY
441 * Generic version of the affinity autoselector.
443 int irq_setup_affinity(struct irq_desc *desc)
445 struct cpumask *set = irq_default_affinity;
446 int ret, node = irq_desc_get_node(desc);
447 static DEFINE_RAW_SPINLOCK(mask_lock);
448 static struct cpumask mask;
450 /* Excludes PER_CPU and NO_BALANCE interrupts */
451 if (!__irq_can_set_affinity(desc))
454 raw_spin_lock(&mask_lock);
456 * Preserve the managed affinity setting and a userspace affinity
457 * setup, but make sure that one of the targets is online.
459 if (irqd_affinity_is_managed(&desc->irq_data) ||
460 irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
461 if (cpumask_intersects(desc->irq_common_data.affinity,
463 set = desc->irq_common_data.affinity;
465 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
468 cpumask_and(&mask, cpu_online_mask, set);
469 if (cpumask_empty(&mask))
470 cpumask_copy(&mask, cpu_online_mask);
472 if (node != NUMA_NO_NODE) {
473 const struct cpumask *nodemask = cpumask_of_node(node);
475 /* make sure at least one of the cpus in nodemask is online */
476 if (cpumask_intersects(&mask, nodemask))
477 cpumask_and(&mask, &mask, nodemask);
479 ret = irq_do_set_affinity(&desc->irq_data, &mask, false);
480 raw_spin_unlock(&mask_lock);
484 /* Wrapper for ALPHA specific affinity selector magic */
485 int irq_setup_affinity(struct irq_desc *desc)
487 return irq_select_affinity(irq_desc_get_irq(desc));
489 #endif /* CONFIG_AUTO_IRQ_AFFINITY */
490 #endif /* CONFIG_SMP */
494 * irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
495 * @irq: interrupt number to set affinity
496 * @vcpu_info: vCPU specific data or pointer to a percpu array of vCPU
497 * specific data for percpu_devid interrupts
499 * This function uses the vCPU specific data to set the vCPU
500 * affinity for an irq. The vCPU specific data is passed from
501 * outside, such as KVM. One example code path is as below:
502 * KVM -> IOMMU -> irq_set_vcpu_affinity().
504 int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
507 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
508 struct irq_data *data;
509 struct irq_chip *chip;
515 data = irq_desc_get_irq_data(desc);
517 chip = irq_data_get_irq_chip(data);
518 if (chip && chip->irq_set_vcpu_affinity)
520 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
521 data = data->parent_data;
528 ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
529 irq_put_desc_unlock(desc, flags);
533 EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
535 void __disable_irq(struct irq_desc *desc)
541 static int __disable_irq_nosync(unsigned int irq)
544 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
549 irq_put_desc_busunlock(desc, flags);
554 * disable_irq_nosync - disable an irq without waiting
555 * @irq: Interrupt to disable
557 * Disable the selected interrupt line. Disables and Enables are
559 * Unlike disable_irq(), this function does not ensure existing
560 * instances of the IRQ handler have completed before returning.
562 * This function may be called from IRQ context.
564 void disable_irq_nosync(unsigned int irq)
566 __disable_irq_nosync(irq);
568 EXPORT_SYMBOL(disable_irq_nosync);
571 * disable_irq - disable an irq and wait for completion
572 * @irq: Interrupt to disable
574 * Disable the selected interrupt line. Enables and Disables are
576 * This function waits for any pending IRQ handlers for this interrupt
577 * to complete before returning. If you use this function while
578 * holding a resource the IRQ handler may need you will deadlock.
580 * This function may be called - with care - from IRQ context.
582 void disable_irq(unsigned int irq)
584 if (!__disable_irq_nosync(irq))
585 synchronize_irq(irq);
587 EXPORT_SYMBOL(disable_irq);
590 * disable_hardirq - disables an irq and waits for hardirq completion
591 * @irq: Interrupt to disable
593 * Disable the selected interrupt line. Enables and Disables are
595 * This function waits for any pending hard IRQ handlers for this
596 * interrupt to complete before returning. If you use this function while
597 * holding a resource the hard IRQ handler may need you will deadlock.
599 * When used to optimistically disable an interrupt from atomic context
600 * the return value must be checked.
602 * Returns: false if a threaded handler is active.
604 * This function may be called - with care - from IRQ context.
606 bool disable_hardirq(unsigned int irq)
608 if (!__disable_irq_nosync(irq))
609 return synchronize_hardirq(irq);
613 EXPORT_SYMBOL_GPL(disable_hardirq);
616 * disable_nmi_nosync - disable an nmi without waiting
617 * @irq: Interrupt to disable
619 * Disable the selected interrupt line. Disables and enables are
621 * The interrupt to disable must have been requested through request_nmi.
622 * Unlike disable_nmi(), this function does not ensure existing
623 * instances of the IRQ handler have completed before returning.
625 void disable_nmi_nosync(unsigned int irq)
627 disable_irq_nosync(irq);
630 void __enable_irq(struct irq_desc *desc)
632 switch (desc->depth) {
635 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
636 irq_desc_get_irq(desc));
639 if (desc->istate & IRQS_SUSPENDED)
641 /* Prevent probing on this irq: */
642 irq_settings_set_noprobe(desc);
644 * Call irq_startup() not irq_enable() here because the
645 * interrupt might be marked NOAUTOEN. So irq_startup()
646 * needs to be invoked when it gets enabled the first
647 * time. If it was already started up, then irq_startup()
648 * will invoke irq_enable() under the hood.
650 irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
659 * enable_irq - enable handling of an irq
660 * @irq: Interrupt to enable
662 * Undoes the effect of one call to disable_irq(). If this
663 * matches the last disable, processing of interrupts on this
664 * IRQ line is re-enabled.
666 * This function may be called from IRQ context only when
667 * desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
669 void enable_irq(unsigned int irq)
672 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
676 if (WARN(!desc->irq_data.chip,
677 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
682 irq_put_desc_busunlock(desc, flags);
684 EXPORT_SYMBOL(enable_irq);
687 * enable_nmi - enable handling of an nmi
688 * @irq: Interrupt to enable
690 * The interrupt to enable must have been requested through request_nmi.
691 * Undoes the effect of one call to disable_nmi(). If this
692 * matches the last disable, processing of interrupts on this
693 * IRQ line is re-enabled.
695 void enable_nmi(unsigned int irq)
700 static int set_irq_wake_real(unsigned int irq, unsigned int on)
702 struct irq_desc *desc = irq_to_desc(irq);
705 if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE)
708 if (desc->irq_data.chip->irq_set_wake)
709 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
715 * irq_set_irq_wake - control irq power management wakeup
716 * @irq: interrupt to control
717 * @on: enable/disable power management wakeup
719 * Enable/disable power management wakeup mode, which is
720 * disabled by default. Enables and disables must match,
721 * just as they match for non-wakeup mode support.
723 * Wakeup mode lets this IRQ wake the system from sleep
724 * states like "suspend to RAM".
726 int irq_set_irq_wake(unsigned int irq, unsigned int on)
729 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
735 /* Don't use NMIs as wake up interrupts please */
736 if (desc->istate & IRQS_NMI) {
741 /* wakeup-capable irqs can be shared between drivers that
742 * don't need to have the same sleep mode behaviors.
745 if (desc->wake_depth++ == 0) {
746 ret = set_irq_wake_real(irq, on);
748 desc->wake_depth = 0;
750 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
753 if (desc->wake_depth == 0) {
754 WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
755 } else if (--desc->wake_depth == 0) {
756 ret = set_irq_wake_real(irq, on);
758 desc->wake_depth = 1;
760 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
765 irq_put_desc_busunlock(desc, flags);
768 EXPORT_SYMBOL(irq_set_irq_wake);
771 * Internal function that tells the architecture code whether a
772 * particular irq has been exclusively allocated or is available
775 int can_request_irq(unsigned int irq, unsigned long irqflags)
778 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
784 if (irq_settings_can_request(desc)) {
786 irqflags & desc->action->flags & IRQF_SHARED)
789 irq_put_desc_unlock(desc, flags);
793 int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
795 struct irq_chip *chip = desc->irq_data.chip;
798 if (!chip || !chip->irq_set_type) {
800 * IRQF_TRIGGER_* but the PIC does not support multiple
803 pr_debug("No set_type function for IRQ %d (%s)\n",
804 irq_desc_get_irq(desc),
805 chip ? (chip->name ? : "unknown") : "unknown");
809 if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
810 if (!irqd_irq_masked(&desc->irq_data))
812 if (!irqd_irq_disabled(&desc->irq_data))
816 /* Mask all flags except trigger mode */
817 flags &= IRQ_TYPE_SENSE_MASK;
818 ret = chip->irq_set_type(&desc->irq_data, flags);
821 case IRQ_SET_MASK_OK:
822 case IRQ_SET_MASK_OK_DONE:
823 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
824 irqd_set(&desc->irq_data, flags);
827 case IRQ_SET_MASK_OK_NOCOPY:
828 flags = irqd_get_trigger_type(&desc->irq_data);
829 irq_settings_set_trigger_mask(desc, flags);
830 irqd_clear(&desc->irq_data, IRQD_LEVEL);
831 irq_settings_clr_level(desc);
832 if (flags & IRQ_TYPE_LEVEL_MASK) {
833 irq_settings_set_level(desc);
834 irqd_set(&desc->irq_data, IRQD_LEVEL);
840 pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
841 flags, irq_desc_get_irq(desc), chip->irq_set_type);
848 #ifdef CONFIG_HARDIRQS_SW_RESEND
849 int irq_set_parent(int irq, int parent_irq)
852 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
857 desc->parent_irq = parent_irq;
859 irq_put_desc_unlock(desc, flags);
862 EXPORT_SYMBOL_GPL(irq_set_parent);
866 * Default primary interrupt handler for threaded interrupts. Is
867 * assigned as primary handler when request_threaded_irq is called
868 * with handler == NULL. Useful for oneshot interrupts.
870 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
872 return IRQ_WAKE_THREAD;
876 * Primary handler for nested threaded interrupts. Should never be
879 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
881 WARN(1, "Primary handler called for nested irq %d\n", irq);
885 static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
887 WARN(1, "Secondary action handler called for irq %d\n", irq);
891 static int irq_wait_for_interrupt(struct irqaction *action)
894 set_current_state(TASK_INTERRUPTIBLE);
896 if (kthread_should_stop()) {
897 /* may need to run one last time */
898 if (test_and_clear_bit(IRQTF_RUNTHREAD,
899 &action->thread_flags)) {
900 __set_current_state(TASK_RUNNING);
903 __set_current_state(TASK_RUNNING);
907 if (test_and_clear_bit(IRQTF_RUNTHREAD,
908 &action->thread_flags)) {
909 __set_current_state(TASK_RUNNING);
917 * Oneshot interrupts keep the irq line masked until the threaded
918 * handler finished. unmask if the interrupt has not been disabled and
921 static void irq_finalize_oneshot(struct irq_desc *desc,
922 struct irqaction *action)
924 if (!(desc->istate & IRQS_ONESHOT) ||
925 action->handler == irq_forced_secondary_handler)
929 raw_spin_lock_irq(&desc->lock);
932 * Implausible though it may be we need to protect us against
933 * the following scenario:
935 * The thread is faster done than the hard interrupt handler
936 * on the other CPU. If we unmask the irq line then the
937 * interrupt can come in again and masks the line, leaves due
938 * to IRQS_INPROGRESS and the irq line is masked forever.
940 * This also serializes the state of shared oneshot handlers
941 * versus "desc->threads_onehsot |= action->thread_mask;" in
942 * irq_wake_thread(). See the comment there which explains the
945 if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
946 raw_spin_unlock_irq(&desc->lock);
947 chip_bus_sync_unlock(desc);
953 * Now check again, whether the thread should run. Otherwise
954 * we would clear the threads_oneshot bit of this thread which
957 if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
960 desc->threads_oneshot &= ~action->thread_mask;
962 if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
963 irqd_irq_masked(&desc->irq_data))
964 unmask_threaded_irq(desc);
967 raw_spin_unlock_irq(&desc->lock);
968 chip_bus_sync_unlock(desc);
973 * Check whether we need to change the affinity of the interrupt thread.
976 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
981 if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
985 * In case we are out of memory we set IRQTF_AFFINITY again and
986 * try again next time
988 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
989 set_bit(IRQTF_AFFINITY, &action->thread_flags);
993 raw_spin_lock_irq(&desc->lock);
995 * This code is triggered unconditionally. Check the affinity
996 * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
998 if (cpumask_available(desc->irq_common_data.affinity)) {
999 const struct cpumask *m;
1001 m = irq_data_get_effective_affinity_mask(&desc->irq_data);
1002 cpumask_copy(mask, m);
1006 raw_spin_unlock_irq(&desc->lock);
1009 set_cpus_allowed_ptr(current, mask);
1010 free_cpumask_var(mask);
1014 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
1018 * Interrupts which are not explicitly requested as threaded
1019 * interrupts rely on the implicit bh/preempt disable of the hard irq
1020 * context. So we need to disable bh here to avoid deadlocks and other
1024 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
1029 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1030 local_irq_disable();
1031 ret = action->thread_fn(action->irq, action->dev_id);
1032 if (ret == IRQ_HANDLED)
1033 atomic_inc(&desc->threads_handled);
1035 irq_finalize_oneshot(desc, action);
1036 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1043 * Interrupts explicitly requested as threaded interrupts want to be
1044 * preemtible - many of them need to sleep and wait for slow busses to
1047 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
1048 struct irqaction *action)
1052 ret = action->thread_fn(action->irq, action->dev_id);
1053 if (ret == IRQ_HANDLED)
1054 atomic_inc(&desc->threads_handled);
1056 irq_finalize_oneshot(desc, action);
1060 static void wake_threads_waitq(struct irq_desc *desc)
1062 if (atomic_dec_and_test(&desc->threads_active))
1063 wake_up(&desc->wait_for_threads);
1066 static void irq_thread_dtor(struct callback_head *unused)
1068 struct task_struct *tsk = current;
1069 struct irq_desc *desc;
1070 struct irqaction *action;
1072 if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
1075 action = kthread_data(tsk);
1077 pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
1078 tsk->comm, tsk->pid, action->irq);
1081 desc = irq_to_desc(action->irq);
1083 * If IRQTF_RUNTHREAD is set, we need to decrement
1084 * desc->threads_active and wake possible waiters.
1086 if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
1087 wake_threads_waitq(desc);
1089 /* Prevent a stale desc->threads_oneshot */
1090 irq_finalize_oneshot(desc, action);
1093 static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
1095 struct irqaction *secondary = action->secondary;
1097 if (WARN_ON_ONCE(!secondary))
1100 raw_spin_lock_irq(&desc->lock);
1101 __irq_wake_thread(desc, secondary);
1102 raw_spin_unlock_irq(&desc->lock);
1106 * Internal function to notify that a interrupt thread is ready.
1108 static void irq_thread_set_ready(struct irq_desc *desc,
1109 struct irqaction *action)
1111 set_bit(IRQTF_READY, &action->thread_flags);
1112 wake_up(&desc->wait_for_threads);
1116 * Internal function to wake up a interrupt thread and wait until it is
1119 static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc,
1120 struct irqaction *action)
1122 if (!action || !action->thread)
1125 wake_up_process(action->thread);
1126 wait_event(desc->wait_for_threads,
1127 test_bit(IRQTF_READY, &action->thread_flags));
1131 * Interrupt handler thread
1133 static int irq_thread(void *data)
1135 struct callback_head on_exit_work;
1136 struct irqaction *action = data;
1137 struct irq_desc *desc = irq_to_desc(action->irq);
1138 irqreturn_t (*handler_fn)(struct irq_desc *desc,
1139 struct irqaction *action);
1141 irq_thread_set_ready(desc, action);
1143 if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
1144 &action->thread_flags))
1145 handler_fn = irq_forced_thread_fn;
1147 handler_fn = irq_thread_fn;
1149 init_task_work(&on_exit_work, irq_thread_dtor);
1150 task_work_add(current, &on_exit_work, false);
1152 irq_thread_check_affinity(desc, action);
1154 while (!irq_wait_for_interrupt(action)) {
1155 irqreturn_t action_ret;
1157 irq_thread_check_affinity(desc, action);
1159 action_ret = handler_fn(desc, action);
1160 if (action_ret == IRQ_WAKE_THREAD)
1161 irq_wake_secondary(desc, action);
1163 wake_threads_waitq(desc);
1167 * This is the regular exit path. __free_irq() is stopping the
1168 * thread via kthread_stop() after calling
1169 * synchronize_hardirq(). So neither IRQTF_RUNTHREAD nor the
1170 * oneshot mask bit can be set.
1172 task_work_cancel(current, irq_thread_dtor);
1177 * irq_wake_thread - wake the irq thread for the action identified by dev_id
1178 * @irq: Interrupt line
1179 * @dev_id: Device identity for which the thread should be woken
1182 void irq_wake_thread(unsigned int irq, void *dev_id)
1184 struct irq_desc *desc = irq_to_desc(irq);
1185 struct irqaction *action;
1186 unsigned long flags;
1188 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1191 raw_spin_lock_irqsave(&desc->lock, flags);
1192 for_each_action_of_desc(desc, action) {
1193 if (action->dev_id == dev_id) {
1195 __irq_wake_thread(desc, action);
1199 raw_spin_unlock_irqrestore(&desc->lock, flags);
1201 EXPORT_SYMBOL_GPL(irq_wake_thread);
1203 static int irq_setup_forced_threading(struct irqaction *new)
1205 if (!force_irqthreads)
1207 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1211 * No further action required for interrupts which are requested as
1212 * threaded interrupts already
1214 if (new->handler == irq_default_primary_handler)
1217 new->flags |= IRQF_ONESHOT;
1220 * Handle the case where we have a real primary handler and a
1221 * thread handler. We force thread them as well by creating a
1224 if (new->handler && new->thread_fn) {
1225 /* Allocate the secondary action */
1226 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1227 if (!new->secondary)
1229 new->secondary->handler = irq_forced_secondary_handler;
1230 new->secondary->thread_fn = new->thread_fn;
1231 new->secondary->dev_id = new->dev_id;
1232 new->secondary->irq = new->irq;
1233 new->secondary->name = new->name;
1235 /* Deal with the primary handler */
1236 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1237 new->thread_fn = new->handler;
1238 new->handler = irq_default_primary_handler;
1242 static int irq_request_resources(struct irq_desc *desc)
1244 struct irq_data *d = &desc->irq_data;
1245 struct irq_chip *c = d->chip;
1247 return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1250 static void irq_release_resources(struct irq_desc *desc)
1252 struct irq_data *d = &desc->irq_data;
1253 struct irq_chip *c = d->chip;
1255 if (c->irq_release_resources)
1256 c->irq_release_resources(d);
1259 static bool irq_supports_nmi(struct irq_desc *desc)
1261 struct irq_data *d = irq_desc_get_irq_data(desc);
1263 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1264 /* Only IRQs directly managed by the root irqchip can be set as NMI */
1268 /* Don't support NMIs for chips behind a slow bus */
1269 if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
1272 return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
1275 static int irq_nmi_setup(struct irq_desc *desc)
1277 struct irq_data *d = irq_desc_get_irq_data(desc);
1278 struct irq_chip *c = d->chip;
1280 return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
1283 static void irq_nmi_teardown(struct irq_desc *desc)
1285 struct irq_data *d = irq_desc_get_irq_data(desc);
1286 struct irq_chip *c = d->chip;
1288 if (c->irq_nmi_teardown)
1289 c->irq_nmi_teardown(d);
1293 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1295 struct task_struct *t;
1296 struct sched_param param = {
1297 .sched_priority = MAX_USER_RT_PRIO/2,
1301 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1304 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1306 param.sched_priority -= 1;
1312 sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m);
1315 * We keep the reference to the task struct even if
1316 * the thread dies to avoid that the interrupt code
1317 * references an already freed task_struct.
1319 new->thread = get_task_struct(t);
1321 * Tell the thread to set its affinity. This is
1322 * important for shared interrupt handlers as we do
1323 * not invoke setup_affinity() for the secondary
1324 * handlers as everything is already set up. Even for
1325 * interrupts marked with IRQF_NO_BALANCE this is
1326 * correct as we want the thread to move to the cpu(s)
1327 * on which the requesting code placed the interrupt.
1329 set_bit(IRQTF_AFFINITY, &new->thread_flags);
1334 * Internal function to register an irqaction - typically used to
1335 * allocate special interrupts that are part of the architecture.
1339 * desc->request_mutex Provides serialization against a concurrent free_irq()
1340 * chip_bus_lock Provides serialization for slow bus operations
1341 * desc->lock Provides serialization against hard interrupts
1343 * chip_bus_lock and desc->lock are sufficient for all other management and
1344 * interrupt related functions. desc->request_mutex solely serializes
1345 * request/free_irq().
1348 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1350 struct irqaction *old, **old_ptr;
1351 unsigned long flags, thread_mask = 0;
1352 int ret, nested, shared = 0;
1357 if (desc->irq_data.chip == &no_irq_chip)
1359 if (!try_module_get(desc->owner))
1365 * If the trigger type is not specified by the caller,
1366 * then use the default for this interrupt.
1368 if (!(new->flags & IRQF_TRIGGER_MASK))
1369 new->flags |= irqd_get_trigger_type(&desc->irq_data);
1372 * Check whether the interrupt nests into another interrupt
1375 nested = irq_settings_is_nested_thread(desc);
1377 if (!new->thread_fn) {
1382 * Replace the primary handler which was provided from
1383 * the driver for non nested interrupt handling by the
1384 * dummy function which warns when called.
1386 new->handler = irq_nested_primary_handler;
1388 if (irq_settings_can_thread(desc)) {
1389 ret = irq_setup_forced_threading(new);
1396 * Create a handler thread when a thread function is supplied
1397 * and the interrupt does not nest into another interrupt
1400 if (new->thread_fn && !nested) {
1401 ret = setup_irq_thread(new, irq, false);
1404 if (new->secondary) {
1405 ret = setup_irq_thread(new->secondary, irq, true);
1412 * Drivers are often written to work w/o knowledge about the
1413 * underlying irq chip implementation, so a request for a
1414 * threaded irq without a primary hard irq context handler
1415 * requires the ONESHOT flag to be set. Some irq chips like
1416 * MSI based interrupts are per se one shot safe. Check the
1417 * chip flags, so we can avoid the unmask dance at the end of
1418 * the threaded handler for those.
1420 if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1421 new->flags &= ~IRQF_ONESHOT;
1424 * Protects against a concurrent __free_irq() call which might wait
1425 * for synchronize_hardirq() to complete without holding the optional
1426 * chip bus lock and desc->lock. Also protects against handing out
1427 * a recycled oneshot thread_mask bit while it's still in use by
1428 * its previous owner.
1430 mutex_lock(&desc->request_mutex);
1433 * Acquire bus lock as the irq_request_resources() callback below
1434 * might rely on the serialization or the magic power management
1435 * functions which are abusing the irq_bus_lock() callback,
1437 chip_bus_lock(desc);
1439 /* First installed action requests resources. */
1440 if (!desc->action) {
1441 ret = irq_request_resources(desc);
1443 pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1444 new->name, irq, desc->irq_data.chip->name);
1445 goto out_bus_unlock;
1450 * The following block of code has to be executed atomically
1451 * protected against a concurrent interrupt and any of the other
1452 * management calls which are not serialized via
1453 * desc->request_mutex or the optional bus lock.
1455 raw_spin_lock_irqsave(&desc->lock, flags);
1456 old_ptr = &desc->action;
1460 * Can't share interrupts unless both agree to and are
1461 * the same type (level, edge, polarity). So both flag
1462 * fields must have IRQF_SHARED set and the bits which
1463 * set the trigger type must match. Also all must
1465 * Interrupt lines used for NMIs cannot be shared.
1467 unsigned int oldtype;
1469 if (desc->istate & IRQS_NMI) {
1470 pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
1471 new->name, irq, desc->irq_data.chip->name);
1477 * If nobody did set the configuration before, inherit
1478 * the one provided by the requester.
1480 if (irqd_trigger_type_was_set(&desc->irq_data)) {
1481 oldtype = irqd_get_trigger_type(&desc->irq_data);
1483 oldtype = new->flags & IRQF_TRIGGER_MASK;
1484 irqd_set_trigger_type(&desc->irq_data, oldtype);
1487 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1488 (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
1489 ((old->flags ^ new->flags) & IRQF_ONESHOT))
1492 /* All handlers must agree on per-cpuness */
1493 if ((old->flags & IRQF_PERCPU) !=
1494 (new->flags & IRQF_PERCPU))
1497 /* add new interrupt at end of irq queue */
1500 * Or all existing action->thread_mask bits,
1501 * so we can find the next zero bit for this
1504 thread_mask |= old->thread_mask;
1505 old_ptr = &old->next;
1512 * Setup the thread mask for this irqaction for ONESHOT. For
1513 * !ONESHOT irqs the thread mask is 0 so we can avoid a
1514 * conditional in irq_wake_thread().
1516 if (new->flags & IRQF_ONESHOT) {
1518 * Unlikely to have 32 resp 64 irqs sharing one line,
1521 if (thread_mask == ~0UL) {
1526 * The thread_mask for the action is or'ed to
1527 * desc->thread_active to indicate that the
1528 * IRQF_ONESHOT thread handler has been woken, but not
1529 * yet finished. The bit is cleared when a thread
1530 * completes. When all threads of a shared interrupt
1531 * line have completed desc->threads_active becomes
1532 * zero and the interrupt line is unmasked. See
1533 * handle.c:irq_wake_thread() for further information.
1535 * If no thread is woken by primary (hard irq context)
1536 * interrupt handlers, then desc->threads_active is
1537 * also checked for zero to unmask the irq line in the
1538 * affected hard irq flow handlers
1539 * (handle_[fasteoi|level]_irq).
1541 * The new action gets the first zero bit of
1542 * thread_mask assigned. See the loop above which or's
1543 * all existing action->thread_mask bits.
1545 new->thread_mask = 1UL << ffz(thread_mask);
1547 } else if (new->handler == irq_default_primary_handler &&
1548 !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1550 * The interrupt was requested with handler = NULL, so
1551 * we use the default primary handler for it. But it
1552 * does not have the oneshot flag set. In combination
1553 * with level interrupts this is deadly, because the
1554 * default primary handler just wakes the thread, then
1555 * the irq lines is reenabled, but the device still
1556 * has the level irq asserted. Rinse and repeat....
1558 * While this works for edge type interrupts, we play
1559 * it safe and reject unconditionally because we can't
1560 * say for sure which type this interrupt really
1561 * has. The type flags are unreliable as the
1562 * underlying chip implementation can override them.
1564 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1571 /* Setup the type (level, edge polarity) if configured: */
1572 if (new->flags & IRQF_TRIGGER_MASK) {
1573 ret = __irq_set_trigger(desc,
1574 new->flags & IRQF_TRIGGER_MASK);
1581 * Activate the interrupt. That activation must happen
1582 * independently of IRQ_NOAUTOEN. request_irq() can fail
1583 * and the callers are supposed to handle
1584 * that. enable_irq() of an interrupt requested with
1585 * IRQ_NOAUTOEN is not supposed to fail. The activation
1586 * keeps it in shutdown mode, it merily associates
1587 * resources if necessary and if that's not possible it
1588 * fails. Interrupts which are in managed shutdown mode
1589 * will simply ignore that activation request.
1591 ret = irq_activate(desc);
1595 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1596 IRQS_ONESHOT | IRQS_WAITING);
1597 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1599 if (new->flags & IRQF_PERCPU) {
1600 irqd_set(&desc->irq_data, IRQD_PER_CPU);
1601 irq_settings_set_per_cpu(desc);
1604 if (new->flags & IRQF_ONESHOT)
1605 desc->istate |= IRQS_ONESHOT;
1607 /* Exclude IRQ from balancing if requested */
1608 if (new->flags & IRQF_NOBALANCING) {
1609 irq_settings_set_no_balancing(desc);
1610 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1613 if (!(new->flags & IRQF_NO_AUTOEN) &&
1614 irq_settings_can_autoenable(desc)) {
1615 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
1618 * Shared interrupts do not go well with disabling
1619 * auto enable. The sharing interrupt might request
1620 * it while it's still disabled and then wait for
1621 * interrupts forever.
1623 WARN_ON_ONCE(new->flags & IRQF_SHARED);
1624 /* Undo nested disables: */
1628 } else if (new->flags & IRQF_TRIGGER_MASK) {
1629 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1630 unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
1633 /* hope the handler works with current trigger mode */
1634 pr_warn("irq %d uses trigger mode %u; requested %u\n",
1640 irq_pm_install_action(desc, new);
1642 /* Reset broken irq detection when installing new handler */
1643 desc->irq_count = 0;
1644 desc->irqs_unhandled = 0;
1647 * Check whether we disabled the irq via the spurious handler
1648 * before. Reenable it and give it another chance.
1650 if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1651 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1655 raw_spin_unlock_irqrestore(&desc->lock, flags);
1656 chip_bus_sync_unlock(desc);
1657 mutex_unlock(&desc->request_mutex);
1659 irq_setup_timings(desc, new);
1661 wake_up_and_wait_for_irq_thread_ready(desc, new);
1662 wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
1664 register_irq_proc(irq, desc);
1666 register_handler_proc(irq, new);
1670 if (!(new->flags & IRQF_PROBE_SHARED)) {
1671 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1672 irq, new->flags, new->name, old->flags, old->name);
1673 #ifdef CONFIG_DEBUG_SHIRQ
1680 raw_spin_unlock_irqrestore(&desc->lock, flags);
1683 irq_release_resources(desc);
1685 chip_bus_sync_unlock(desc);
1686 mutex_unlock(&desc->request_mutex);
1690 struct task_struct *t = new->thread;
1696 if (new->secondary && new->secondary->thread) {
1697 struct task_struct *t = new->secondary->thread;
1699 new->secondary->thread = NULL;
1704 module_put(desc->owner);
1709 * setup_irq - setup an interrupt
1710 * @irq: Interrupt line to setup
1711 * @act: irqaction for the interrupt
1713 * Used to statically setup interrupts in the early boot process.
1715 int setup_irq(unsigned int irq, struct irqaction *act)
1718 struct irq_desc *desc = irq_to_desc(irq);
1720 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1723 retval = irq_chip_pm_get(&desc->irq_data);
1727 retval = __setup_irq(irq, desc, act);
1730 irq_chip_pm_put(&desc->irq_data);
1734 EXPORT_SYMBOL_GPL(setup_irq);
1737 * Internal function to unregister an irqaction - used to free
1738 * regular and special interrupts that are part of the architecture.
1740 static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
1742 unsigned irq = desc->irq_data.irq;
1743 struct irqaction *action, **action_ptr;
1744 unsigned long flags;
1746 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1748 mutex_lock(&desc->request_mutex);
1749 chip_bus_lock(desc);
1750 raw_spin_lock_irqsave(&desc->lock, flags);
1753 * There can be multiple actions per IRQ descriptor, find the right
1754 * one based on the dev_id:
1756 action_ptr = &desc->action;
1758 action = *action_ptr;
1761 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1762 raw_spin_unlock_irqrestore(&desc->lock, flags);
1763 chip_bus_sync_unlock(desc);
1764 mutex_unlock(&desc->request_mutex);
1768 if (action->dev_id == dev_id)
1770 action_ptr = &action->next;
1773 /* Found it - now remove it from the list of entries: */
1774 *action_ptr = action->next;
1776 irq_pm_remove_action(desc, action);
1778 /* If this was the last handler, shut down the IRQ line: */
1779 if (!desc->action) {
1780 irq_settings_clr_disable_unlazy(desc);
1781 /* Only shutdown. Deactivate after synchronize_hardirq() */
1786 /* make sure affinity_hint is cleaned up */
1787 if (WARN_ON_ONCE(desc->affinity_hint))
1788 desc->affinity_hint = NULL;
1791 raw_spin_unlock_irqrestore(&desc->lock, flags);
1793 * Drop bus_lock here so the changes which were done in the chip
1794 * callbacks above are synced out to the irq chips which hang
1795 * behind a slow bus (I2C, SPI) before calling synchronize_hardirq().
1797 * Aside of that the bus_lock can also be taken from the threaded
1798 * handler in irq_finalize_oneshot() which results in a deadlock
1799 * because kthread_stop() would wait forever for the thread to
1800 * complete, which is blocked on the bus lock.
1802 * The still held desc->request_mutex() protects against a
1803 * concurrent request_irq() of this irq so the release of resources
1804 * and timing data is properly serialized.
1806 chip_bus_sync_unlock(desc);
1808 unregister_handler_proc(irq, action);
1811 * Make sure it's not being used on another CPU and if the chip
1812 * supports it also make sure that there is no (not yet serviced)
1813 * interrupt in flight at the hardware level.
1815 __synchronize_hardirq(desc, true);
1817 #ifdef CONFIG_DEBUG_SHIRQ
1819 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1820 * event to happen even now it's being freed, so let's make sure that
1821 * is so by doing an extra call to the handler ....
1823 * ( We do this after actually deregistering it, to make sure that a
1824 * 'real' IRQ doesn't run in parallel with our fake. )
1826 if (action->flags & IRQF_SHARED) {
1827 local_irq_save(flags);
1828 action->handler(irq, dev_id);
1829 local_irq_restore(flags);
1834 * The action has already been removed above, but the thread writes
1835 * its oneshot mask bit when it completes. Though request_mutex is
1836 * held across this which prevents __setup_irq() from handing out
1837 * the same bit to a newly requested action.
1839 if (action->thread) {
1840 kthread_stop(action->thread);
1841 put_task_struct(action->thread);
1842 if (action->secondary && action->secondary->thread) {
1843 kthread_stop(action->secondary->thread);
1844 put_task_struct(action->secondary->thread);
1848 /* Last action releases resources */
1849 if (!desc->action) {
1851 * Reaquire bus lock as irq_release_resources() might
1852 * require it to deallocate resources over the slow bus.
1854 chip_bus_lock(desc);
1856 * There is no interrupt on the fly anymore. Deactivate it
1859 raw_spin_lock_irqsave(&desc->lock, flags);
1860 irq_domain_deactivate_irq(&desc->irq_data);
1861 raw_spin_unlock_irqrestore(&desc->lock, flags);
1863 irq_release_resources(desc);
1864 chip_bus_sync_unlock(desc);
1865 irq_remove_timings(desc);
1868 mutex_unlock(&desc->request_mutex);
1870 irq_chip_pm_put(&desc->irq_data);
1871 module_put(desc->owner);
1872 kfree(action->secondary);
1877 * remove_irq - free an interrupt
1878 * @irq: Interrupt line to free
1879 * @act: irqaction for the interrupt
1881 * Used to remove interrupts statically setup by the early boot process.
1883 void remove_irq(unsigned int irq, struct irqaction *act)
1885 struct irq_desc *desc = irq_to_desc(irq);
1887 if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1888 __free_irq(desc, act->dev_id);
1890 EXPORT_SYMBOL_GPL(remove_irq);
1893 * free_irq - free an interrupt allocated with request_irq
1894 * @irq: Interrupt line to free
1895 * @dev_id: Device identity to free
1897 * Remove an interrupt handler. The handler is removed and if the
1898 * interrupt line is no longer in use by any driver it is disabled.
1899 * On a shared IRQ the caller must ensure the interrupt is disabled
1900 * on the card it drives before calling this function. The function
1901 * does not return until any executing interrupts for this IRQ
1904 * This function must not be called from interrupt context.
1906 * Returns the devname argument passed to request_irq.
1908 const void *free_irq(unsigned int irq, void *dev_id)
1910 struct irq_desc *desc = irq_to_desc(irq);
1911 struct irqaction *action;
1912 const char *devname;
1914 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1918 if (WARN_ON(desc->affinity_notify))
1919 desc->affinity_notify = NULL;
1922 action = __free_irq(desc, dev_id);
1927 devname = action->name;
1931 EXPORT_SYMBOL(free_irq);
1933 /* This function must be called with desc->lock held */
1934 static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
1936 const char *devname = NULL;
1938 desc->istate &= ~IRQS_NMI;
1940 if (!WARN_ON(desc->action == NULL)) {
1941 irq_pm_remove_action(desc, desc->action);
1942 devname = desc->action->name;
1943 unregister_handler_proc(irq, desc->action);
1945 kfree(desc->action);
1946 desc->action = NULL;
1949 irq_settings_clr_disable_unlazy(desc);
1950 irq_shutdown_and_deactivate(desc);
1952 irq_release_resources(desc);
1954 irq_chip_pm_put(&desc->irq_data);
1955 module_put(desc->owner);
1960 const void *free_nmi(unsigned int irq, void *dev_id)
1962 struct irq_desc *desc = irq_to_desc(irq);
1963 unsigned long flags;
1964 const void *devname;
1966 if (!desc || WARN_ON(!(desc->istate & IRQS_NMI)))
1969 if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1972 /* NMI still enabled */
1973 if (WARN_ON(desc->depth == 0))
1974 disable_nmi_nosync(irq);
1976 raw_spin_lock_irqsave(&desc->lock, flags);
1978 irq_nmi_teardown(desc);
1979 devname = __cleanup_nmi(irq, desc);
1981 raw_spin_unlock_irqrestore(&desc->lock, flags);
1987 * request_threaded_irq - allocate an interrupt line
1988 * @irq: Interrupt line to allocate
1989 * @handler: Function to be called when the IRQ occurs.
1990 * Primary handler for threaded interrupts
1991 * If NULL and thread_fn != NULL the default
1992 * primary handler is installed
1993 * @thread_fn: Function called from the irq handler thread
1994 * If NULL, no irq thread is created
1995 * @irqflags: Interrupt type flags
1996 * @devname: An ascii name for the claiming device
1997 * @dev_id: A cookie passed back to the handler function
1999 * This call allocates interrupt resources and enables the
2000 * interrupt line and IRQ handling. From the point this
2001 * call is made your handler function may be invoked. Since
2002 * your handler function must clear any interrupt the board
2003 * raises, you must take care both to initialise your hardware
2004 * and to set up the interrupt handler in the right order.
2006 * If you want to set up a threaded irq handler for your device
2007 * then you need to supply @handler and @thread_fn. @handler is
2008 * still called in hard interrupt context and has to check
2009 * whether the interrupt originates from the device. If yes it
2010 * needs to disable the interrupt on the device and return
2011 * IRQ_WAKE_THREAD which will wake up the handler thread and run
2012 * @thread_fn. This split handler design is necessary to support
2013 * shared interrupts.
2015 * Dev_id must be globally unique. Normally the address of the
2016 * device data structure is used as the cookie. Since the handler
2017 * receives this value it makes sense to use it.
2019 * If your interrupt is shared you must pass a non NULL dev_id
2020 * as this is required when freeing the interrupt.
2024 * IRQF_SHARED Interrupt is shared
2025 * IRQF_TRIGGER_* Specify active edge(s) or level
2028 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
2029 irq_handler_t thread_fn, unsigned long irqflags,
2030 const char *devname, void *dev_id)
2032 struct irqaction *action;
2033 struct irq_desc *desc;
2036 if (irq == IRQ_NOTCONNECTED)
2040 * Sanity-check: shared interrupts must pass in a real dev-ID,
2041 * otherwise we'll have trouble later trying to figure out
2042 * which interrupt is which (messes up the interrupt freeing
2045 * Also shared interrupts do not go well with disabling auto enable.
2046 * The sharing interrupt might request it while it's still disabled
2047 * and then wait for interrupts forever.
2049 * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
2050 * it cannot be set along with IRQF_NO_SUSPEND.
2052 if (((irqflags & IRQF_SHARED) && !dev_id) ||
2053 ((irqflags & IRQF_SHARED) && (irqflags & IRQF_NO_AUTOEN)) ||
2054 (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
2055 ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
2058 desc = irq_to_desc(irq);
2062 if (!irq_settings_can_request(desc) ||
2063 WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2069 handler = irq_default_primary_handler;
2072 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2076 action->handler = handler;
2077 action->thread_fn = thread_fn;
2078 action->flags = irqflags;
2079 action->name = devname;
2080 action->dev_id = dev_id;
2082 retval = irq_chip_pm_get(&desc->irq_data);
2088 retval = __setup_irq(irq, desc, action);
2091 irq_chip_pm_put(&desc->irq_data);
2092 kfree(action->secondary);
2096 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
2097 if (!retval && (irqflags & IRQF_SHARED)) {
2099 * It's a shared IRQ -- the driver ought to be prepared for it
2100 * to happen immediately, so let's make sure....
2101 * We disable the irq to make sure that a 'real' IRQ doesn't
2102 * run in parallel with our fake.
2104 unsigned long flags;
2107 local_irq_save(flags);
2109 handler(irq, dev_id);
2111 local_irq_restore(flags);
2117 EXPORT_SYMBOL(request_threaded_irq);
2120 * request_any_context_irq - allocate an interrupt line
2121 * @irq: Interrupt line to allocate
2122 * @handler: Function to be called when the IRQ occurs.
2123 * Threaded handler for threaded interrupts.
2124 * @flags: Interrupt type flags
2125 * @name: An ascii name for the claiming device
2126 * @dev_id: A cookie passed back to the handler function
2128 * This call allocates interrupt resources and enables the
2129 * interrupt line and IRQ handling. It selects either a
2130 * hardirq or threaded handling method depending on the
2133 * On failure, it returns a negative value. On success,
2134 * it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
2136 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
2137 unsigned long flags, const char *name, void *dev_id)
2139 struct irq_desc *desc;
2142 if (irq == IRQ_NOTCONNECTED)
2145 desc = irq_to_desc(irq);
2149 if (irq_settings_is_nested_thread(desc)) {
2150 ret = request_threaded_irq(irq, NULL, handler,
2151 flags, name, dev_id);
2152 return !ret ? IRQC_IS_NESTED : ret;
2155 ret = request_irq(irq, handler, flags, name, dev_id);
2156 return !ret ? IRQC_IS_HARDIRQ : ret;
2158 EXPORT_SYMBOL_GPL(request_any_context_irq);
2161 * request_nmi - allocate an interrupt line for NMI delivery
2162 * @irq: Interrupt line to allocate
2163 * @handler: Function to be called when the IRQ occurs.
2164 * Threaded handler for threaded interrupts.
2165 * @irqflags: Interrupt type flags
2166 * @name: An ascii name for the claiming device
2167 * @dev_id: A cookie passed back to the handler function
2169 * This call allocates interrupt resources and enables the
2170 * interrupt line and IRQ handling. It sets up the IRQ line
2171 * to be handled as an NMI.
2173 * An interrupt line delivering NMIs cannot be shared and IRQ handling
2174 * cannot be threaded.
2176 * Interrupt lines requested for NMI delivering must produce per cpu
2177 * interrupts and have auto enabling setting disabled.
2179 * Dev_id must be globally unique. Normally the address of the
2180 * device data structure is used as the cookie. Since the handler
2181 * receives this value it makes sense to use it.
2183 * If the interrupt line cannot be used to deliver NMIs, function
2184 * will fail and return a negative value.
2186 int request_nmi(unsigned int irq, irq_handler_t handler,
2187 unsigned long irqflags, const char *name, void *dev_id)
2189 struct irqaction *action;
2190 struct irq_desc *desc;
2191 unsigned long flags;
2194 if (irq == IRQ_NOTCONNECTED)
2197 /* NMI cannot be shared, used for Polling */
2198 if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
2201 if (!(irqflags & IRQF_PERCPU))
2207 desc = irq_to_desc(irq);
2209 if (!desc || (irq_settings_can_autoenable(desc) &&
2210 !(irqflags & IRQF_NO_AUTOEN)) ||
2211 !irq_settings_can_request(desc) ||
2212 WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
2213 !irq_supports_nmi(desc))
2216 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2220 action->handler = handler;
2221 action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
2222 action->name = name;
2223 action->dev_id = dev_id;
2225 retval = irq_chip_pm_get(&desc->irq_data);
2229 retval = __setup_irq(irq, desc, action);
2233 raw_spin_lock_irqsave(&desc->lock, flags);
2235 /* Setup NMI state */
2236 desc->istate |= IRQS_NMI;
2237 retval = irq_nmi_setup(desc);
2239 __cleanup_nmi(irq, desc);
2240 raw_spin_unlock_irqrestore(&desc->lock, flags);
2244 raw_spin_unlock_irqrestore(&desc->lock, flags);
2249 irq_chip_pm_put(&desc->irq_data);
2256 void enable_percpu_irq(unsigned int irq, unsigned int type)
2258 unsigned int cpu = smp_processor_id();
2259 unsigned long flags;
2260 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2266 * If the trigger type is not specified by the caller, then
2267 * use the default for this interrupt.
2269 type &= IRQ_TYPE_SENSE_MASK;
2270 if (type == IRQ_TYPE_NONE)
2271 type = irqd_get_trigger_type(&desc->irq_data);
2273 if (type != IRQ_TYPE_NONE) {
2276 ret = __irq_set_trigger(desc, type);
2279 WARN(1, "failed to set type for IRQ%d\n", irq);
2284 irq_percpu_enable(desc, cpu);
2286 irq_put_desc_unlock(desc, flags);
2288 EXPORT_SYMBOL_GPL(enable_percpu_irq);
2290 void enable_percpu_nmi(unsigned int irq, unsigned int type)
2292 enable_percpu_irq(irq, type);
2296 * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
2297 * @irq: Linux irq number to check for
2299 * Must be called from a non migratable context. Returns the enable
2300 * state of a per cpu interrupt on the current cpu.
2302 bool irq_percpu_is_enabled(unsigned int irq)
2304 unsigned int cpu = smp_processor_id();
2305 struct irq_desc *desc;
2306 unsigned long flags;
2309 desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2313 is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
2314 irq_put_desc_unlock(desc, flags);
2318 EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
2320 void disable_percpu_irq(unsigned int irq)
2322 unsigned int cpu = smp_processor_id();
2323 unsigned long flags;
2324 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2329 irq_percpu_disable(desc, cpu);
2330 irq_put_desc_unlock(desc, flags);
2332 EXPORT_SYMBOL_GPL(disable_percpu_irq);
2334 void disable_percpu_nmi(unsigned int irq)
2336 disable_percpu_irq(irq);
2340 * Internal function to unregister a percpu irqaction.
2342 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2344 struct irq_desc *desc = irq_to_desc(irq);
2345 struct irqaction *action;
2346 unsigned long flags;
2348 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
2353 raw_spin_lock_irqsave(&desc->lock, flags);
2355 action = desc->action;
2356 if (!action || action->percpu_dev_id != dev_id) {
2357 WARN(1, "Trying to free already-free IRQ %d\n", irq);
2361 if (!cpumask_empty(desc->percpu_enabled)) {
2362 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
2363 irq, cpumask_first(desc->percpu_enabled));
2367 /* Found it - now remove it from the list of entries: */
2368 desc->action = NULL;
2370 desc->istate &= ~IRQS_NMI;
2372 raw_spin_unlock_irqrestore(&desc->lock, flags);
2374 unregister_handler_proc(irq, action);
2376 irq_chip_pm_put(&desc->irq_data);
2377 module_put(desc->owner);
2381 raw_spin_unlock_irqrestore(&desc->lock, flags);
2386 * remove_percpu_irq - free a per-cpu interrupt
2387 * @irq: Interrupt line to free
2388 * @act: irqaction for the interrupt
2390 * Used to remove interrupts statically setup by the early boot process.
2392 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
2394 struct irq_desc *desc = irq_to_desc(irq);
2396 if (desc && irq_settings_is_per_cpu_devid(desc))
2397 __free_percpu_irq(irq, act->percpu_dev_id);
2401 * free_percpu_irq - free an interrupt allocated with request_percpu_irq
2402 * @irq: Interrupt line to free
2403 * @dev_id: Device identity to free
2405 * Remove a percpu interrupt handler. The handler is removed, but
2406 * the interrupt line is not disabled. This must be done on each
2407 * CPU before calling this function. The function does not return
2408 * until any executing interrupts for this IRQ have completed.
2410 * This function must not be called from interrupt context.
2412 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2414 struct irq_desc *desc = irq_to_desc(irq);
2416 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2419 chip_bus_lock(desc);
2420 kfree(__free_percpu_irq(irq, dev_id));
2421 chip_bus_sync_unlock(desc);
2423 EXPORT_SYMBOL_GPL(free_percpu_irq);
2425 void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
2427 struct irq_desc *desc = irq_to_desc(irq);
2429 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2432 if (WARN_ON(!(desc->istate & IRQS_NMI)))
2435 kfree(__free_percpu_irq(irq, dev_id));
2439 * setup_percpu_irq - setup a per-cpu interrupt
2440 * @irq: Interrupt line to setup
2441 * @act: irqaction for the interrupt
2443 * Used to statically setup per-cpu interrupts in the early boot process.
2445 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
2447 struct irq_desc *desc = irq_to_desc(irq);
2450 if (!desc || !irq_settings_is_per_cpu_devid(desc))
2453 retval = irq_chip_pm_get(&desc->irq_data);
2457 retval = __setup_irq(irq, desc, act);
2460 irq_chip_pm_put(&desc->irq_data);
2466 * __request_percpu_irq - allocate a percpu interrupt line
2467 * @irq: Interrupt line to allocate
2468 * @handler: Function to be called when the IRQ occurs.
2469 * @flags: Interrupt type flags (IRQF_TIMER only)
2470 * @devname: An ascii name for the claiming device
2471 * @dev_id: A percpu cookie passed back to the handler function
2473 * This call allocates interrupt resources and enables the
2474 * interrupt on the local CPU. If the interrupt is supposed to be
2475 * enabled on other CPUs, it has to be done on each CPU using
2476 * enable_percpu_irq().
2478 * Dev_id must be globally unique. It is a per-cpu variable, and
2479 * the handler gets called with the interrupted CPU's instance of
2482 int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
2483 unsigned long flags, const char *devname,
2484 void __percpu *dev_id)
2486 struct irqaction *action;
2487 struct irq_desc *desc;
2493 desc = irq_to_desc(irq);
2494 if (!desc || !irq_settings_can_request(desc) ||
2495 !irq_settings_is_per_cpu_devid(desc))
2498 if (flags && flags != IRQF_TIMER)
2501 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2505 action->handler = handler;
2506 action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND;
2507 action->name = devname;
2508 action->percpu_dev_id = dev_id;
2510 retval = irq_chip_pm_get(&desc->irq_data);
2516 retval = __setup_irq(irq, desc, action);
2519 irq_chip_pm_put(&desc->irq_data);
2525 EXPORT_SYMBOL_GPL(__request_percpu_irq);
2528 * request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
2529 * @irq: Interrupt line to allocate
2530 * @handler: Function to be called when the IRQ occurs.
2531 * @name: An ascii name for the claiming device
2532 * @dev_id: A percpu cookie passed back to the handler function
2534 * This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
2535 * have to be setup on each CPU by calling prepare_percpu_nmi() before
2536 * being enabled on the same CPU by using enable_percpu_nmi().
2538 * Dev_id must be globally unique. It is a per-cpu variable, and
2539 * the handler gets called with the interrupted CPU's instance of
2542 * Interrupt lines requested for NMI delivering should have auto enabling
2545 * If the interrupt line cannot be used to deliver NMIs, function
2546 * will fail returning a negative value.
2548 int request_percpu_nmi(unsigned int irq, irq_handler_t handler,
2549 const char *name, void __percpu *dev_id)
2551 struct irqaction *action;
2552 struct irq_desc *desc;
2553 unsigned long flags;
2559 desc = irq_to_desc(irq);
2561 if (!desc || !irq_settings_can_request(desc) ||
2562 !irq_settings_is_per_cpu_devid(desc) ||
2563 irq_settings_can_autoenable(desc) ||
2564 !irq_supports_nmi(desc))
2567 /* The line cannot already be NMI */
2568 if (desc->istate & IRQS_NMI)
2571 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2575 action->handler = handler;
2576 action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD
2578 action->name = name;
2579 action->percpu_dev_id = dev_id;
2581 retval = irq_chip_pm_get(&desc->irq_data);
2585 retval = __setup_irq(irq, desc, action);
2589 raw_spin_lock_irqsave(&desc->lock, flags);
2590 desc->istate |= IRQS_NMI;
2591 raw_spin_unlock_irqrestore(&desc->lock, flags);
2596 irq_chip_pm_put(&desc->irq_data);
2604 * prepare_percpu_nmi - performs CPU local setup for NMI delivery
2605 * @irq: Interrupt line to prepare for NMI delivery
2607 * This call prepares an interrupt line to deliver NMI on the current CPU,
2608 * before that interrupt line gets enabled with enable_percpu_nmi().
2610 * As a CPU local operation, this should be called from non-preemptible
2613 * If the interrupt line cannot be used to deliver NMIs, function
2614 * will fail returning a negative value.
2616 int prepare_percpu_nmi(unsigned int irq)
2618 unsigned long flags;
2619 struct irq_desc *desc;
2622 WARN_ON(preemptible());
2624 desc = irq_get_desc_lock(irq, &flags,
2625 IRQ_GET_DESC_CHECK_PERCPU);
2629 if (WARN(!(desc->istate & IRQS_NMI),
2630 KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n",
2636 ret = irq_nmi_setup(desc);
2638 pr_err("Failed to setup NMI delivery: irq %u\n", irq);
2643 irq_put_desc_unlock(desc, flags);
2648 * teardown_percpu_nmi - undoes NMI setup of IRQ line
2649 * @irq: Interrupt line from which CPU local NMI configuration should be
2652 * This call undoes the setup done by prepare_percpu_nmi().
2654 * IRQ line should not be enabled for the current CPU.
2656 * As a CPU local operation, this should be called from non-preemptible
2659 void teardown_percpu_nmi(unsigned int irq)
2661 unsigned long flags;
2662 struct irq_desc *desc;
2664 WARN_ON(preemptible());
2666 desc = irq_get_desc_lock(irq, &flags,
2667 IRQ_GET_DESC_CHECK_PERCPU);
2671 if (WARN_ON(!(desc->istate & IRQS_NMI)))
2674 irq_nmi_teardown(desc);
2676 irq_put_desc_unlock(desc, flags);
2679 int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which,
2682 struct irq_chip *chip;
2686 chip = irq_data_get_irq_chip(data);
2687 if (chip->irq_get_irqchip_state)
2689 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2690 data = data->parent_data;
2697 err = chip->irq_get_irqchip_state(data, which, state);
2702 * irq_get_irqchip_state - returns the irqchip state of a interrupt.
2703 * @irq: Interrupt line that is forwarded to a VM
2704 * @which: One of IRQCHIP_STATE_* the caller wants to know about
2705 * @state: a pointer to a boolean where the state is to be storeed
2707 * This call snapshots the internal irqchip state of an
2708 * interrupt, returning into @state the bit corresponding to
2711 * This function should be called with preemption disabled if the
2712 * interrupt controller has per-cpu registers.
2714 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2717 struct irq_desc *desc;
2718 struct irq_data *data;
2719 unsigned long flags;
2722 desc = irq_get_desc_buslock(irq, &flags, 0);
2726 data = irq_desc_get_irq_data(desc);
2728 err = __irq_get_irqchip_state(data, which, state);
2730 irq_put_desc_busunlock(desc, flags);
2733 EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
2736 * irq_set_irqchip_state - set the state of a forwarded interrupt.
2737 * @irq: Interrupt line that is forwarded to a VM
2738 * @which: State to be restored (one of IRQCHIP_STATE_*)
2739 * @val: Value corresponding to @which
2741 * This call sets the internal irqchip state of an interrupt,
2742 * depending on the value of @which.
2744 * This function should be called with preemption disabled if the
2745 * interrupt controller has per-cpu registers.
2747 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2750 struct irq_desc *desc;
2751 struct irq_data *data;
2752 struct irq_chip *chip;
2753 unsigned long flags;
2756 desc = irq_get_desc_buslock(irq, &flags, 0);
2760 data = irq_desc_get_irq_data(desc);
2763 chip = irq_data_get_irq_chip(data);
2764 if (chip->irq_set_irqchip_state)
2766 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2767 data = data->parent_data;
2774 err = chip->irq_set_irqchip_state(data, which, val);
2776 irq_put_desc_busunlock(desc, flags);
2779 EXPORT_SYMBOL_GPL(irq_set_irqchip_state);