kernel/trace/trace_preemptirq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * preemptoff and irqoff tracepoints
   4  *
   5  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
   6  */
   7
   8 #include <linux/kallsyms.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/module.h>
  11 #include <linux/ftrace.h>
  12 #include "trace.h"
  13
  14 #define CREATE_TRACE_POINTS
  15 #include <trace/events/preemptirq.h>
  16
  17 #ifdef CONFIG_TRACE_IRQFLAGS
  18 /* Per-cpu variable to prevent redundant calls when IRQs already off */
  19 static DEFINE_PER_CPU(int, tracing_irq_cpu);
  20
  21 void trace_hardirqs_on(void)
  22 {
  23         if (this_cpu_read(tracing_irq_cpu)) {
  24                 if (!in_nmi())
  25                         trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  26                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  27                 this_cpu_write(tracing_irq_cpu, 0);
  28         }
  29
  30         lockdep_hardirqs_on(CALLER_ADDR0);
  31 }
  32 EXPORT_SYMBOL(trace_hardirqs_on);
  33
  34 void trace_hardirqs_off(void)
  35 {
  36         if (!this_cpu_read(tracing_irq_cpu)) {
  37                 this_cpu_write(tracing_irq_cpu, 1);
  38                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  39                 if (!in_nmi())
  40                         trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  41         }
  42
  43         lockdep_hardirqs_off(CALLER_ADDR0);
  44 }
  45 EXPORT_SYMBOL(trace_hardirqs_off);
  46
  47 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
  48 {
  49         if (this_cpu_read(tracing_irq_cpu)) {
  50                 if (!in_nmi())
  51                         trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
  52                 tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
  53                 this_cpu_write(tracing_irq_cpu, 0);
  54         }
  55
  56         lockdep_hardirqs_on(CALLER_ADDR0);
  57 }
  58 EXPORT_SYMBOL(trace_hardirqs_on_caller);
  59
  60 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
  61 {
  62         lockdep_hardirqs_off(CALLER_ADDR0);
  63
  64         if (!this_cpu_read(tracing_irq_cpu)) {
  65                 this_cpu_write(tracing_irq_cpu, 1);
  66                 tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
  67                 if (!in_nmi())
  68                         trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
  69         }
  70 }
  71 EXPORT_SYMBOL(trace_hardirqs_off_caller);
  72 #endif /* CONFIG_TRACE_IRQFLAGS */
  73
  74 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
  75
  76 void trace_preempt_on(unsigned long a0, unsigned long a1)
  77 {
  78         if (!in_nmi())
  79                 trace_preempt_enable_rcuidle(a0, a1);
  80         tracer_preempt_on(a0, a1);
  81 }
  82
  83 void trace_preempt_off(unsigned long a0, unsigned long a1)
  84 {
  85         if (!in_nmi())
  86                 trace_preempt_disable_rcuidle(a0, a1);
  87         tracer_preempt_off(a0, a1);
  88 }
  89 #endif