arch/x86/include/asm/irq_stack.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_IRQ_STACK_H
   3 #define _ASM_X86_IRQ_STACK_H
   4
   5 #include <linux/ptrace.h>
   6
   7 #include <asm/processor.h>
   8
   9 #ifdef CONFIG_X86_64
  10
  11 /*
  12  * Macro to inline switching to an interrupt stack and invoking function
  13  * calls from there. The following rules apply:
  14  *
  15  * - Ordering:
  16  *
  17  *   1. Write the stack pointer into the top most place of the irq
  18  *      stack. This ensures that the various unwinders can link back to the
  19  *      original stack.
  20  *
  21  *   2. Switch the stack pointer to the top of the irq stack.
  22  *
  23  *   3. Invoke whatever needs to be done (@asm_call argument)
  24  *
  25  *   4. Pop the original stack pointer from the top of the irq stack
  26  *      which brings it back to the original stack where it left off.
  27  *
  28  * - Function invocation:
  29  *
  30  *   To allow flexible usage of the macro, the actual function code including
  31  *   the store of the arguments in the call ABI registers is handed in via
  32  *   the @asm_call argument.
  33  *
  34  * - Local variables:
  35  *
  36  *   @tos:
  37  *      The @tos variable holds a pointer to the top of the irq stack and
  38  *      _must_ be allocated in a non-callee saved register as this is a
  39  *      restriction coming from objtool.
  40  *
  41  *      Note, that (tos) is both in input and output constraints to ensure
  42  *      that the compiler does not assume that R11 is left untouched in
  43  *      case this macro is used in some place where the per cpu interrupt
  44  *      stack pointer is used again afterwards
  45  *
  46  * - Function arguments:
  47  *      The function argument(s), if any, have to be defined in register
  48  *      variables at the place where this is invoked. Storing the
  49  *      argument(s) in the proper register(s) is part of the @asm_call
  50  *
  51  * - Constraints:
  52  *
  53  *   The constraints have to be done very carefully because the compiler
  54  *   does not know about the assembly call.
  55  *
  56  *   output:
  57  *     As documented already above the @tos variable is required to be in
  58  *     the output constraints to make the compiler aware that R11 cannot be
  59  *     reused after the asm() statement.
  60  *
  61  *     For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is
  62  *     required as well as this prevents certain creative GCC variants from
  63  *     misplacing the ASM code.
  64  *
  65  *  input:
  66  *    - func:
  67  *        Immediate, which tells the compiler that the function is referenced.
  68  *
  69  *    - tos:
  70  *        Register. The actual register is defined by the variable declaration.
  71  *
  72  *    - function arguments:
  73  *        The constraints are handed in via the 'argconstr' argument list. They
  74  *        describe the register arguments which are used in @asm_call.
  75  *
  76  *  clobbers:
  77  *     Function calls can clobber anything except the callee-saved
  78  *     registers. Tell the compiler.
  79  */
  80 #define call_on_stack(stack, func, asm_call, argconstr...)              \
  81 {                                                                       \
  82         register void *tos asm("r11");                                  \
  83                                                                         \
  84         tos = ((void *)(stack));                                        \
  85                                                                         \
  86         asm_inline volatile(                                            \
  87         "movq   %%rsp, (%[tos])                         \n"             \
  88         "movq   %[tos], %%rsp                           \n"             \
  89                                                                         \
  90         asm_call                                                        \
  91                                                                         \
  92         "popq   %%rsp                                   \n"             \
  93                                                                         \
  94         : "+r" (tos), ASM_CALL_CONSTRAINT                               \
  95         : [__func] "i" (func), [tos] "r" (tos) argconstr                \
  96         : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10",   \
  97           "memory"                                                      \
  98         );                                                              \
  99 }
 100
 101 #define ASM_CALL_ARG0                                                   \
 102         "call %P[__func]                                \n"             \
 103         ASM_REACHABLE
 104
 105 #define ASM_CALL_ARG1                                                   \
 106         "movq   %[arg1], %%rdi                          \n"             \
 107         ASM_CALL_ARG0
 108
 109 #define ASM_CALL_ARG2                                                   \
 110         "movq   %[arg2], %%rsi                          \n"             \
 111         ASM_CALL_ARG1
 112
 113 #define ASM_CALL_ARG3                                                   \
 114         "movq   %[arg3], %%rdx                          \n"             \
 115         ASM_CALL_ARG2
 116
 117 #define call_on_irqstack(func, asm_call, argconstr...)                  \
 118         call_on_stack(__this_cpu_read(hardirq_stack_ptr),               \
 119                       func, asm_call, argconstr)
 120
 121 /* Macros to assert type correctness for run_*_on_irqstack macros */
 122 #define assert_function_type(func, proto)                               \
 123         static_assert(__builtin_types_compatible_p(typeof(&func), proto))
 124
 125 #define assert_arg_type(arg, proto)                                     \
 126         static_assert(__builtin_types_compatible_p(typeof(arg), proto))
 127
 128 /*
 129  * Macro to invoke system vector and device interrupt C handlers.
 130  */
 131 #define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...)  \
 132 {                                                                       \
 133         /*                                                              \
 134          * User mode entry and interrupt on the irq stack do not        \
 135          * switch stacks. If from user mode the task stack is empty.    \
 136          */                                                             \
 137         if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {  \
 138                 irq_enter_rcu();                                        \
 139                 func(c_args);                                           \
 140                 irq_exit_rcu();                                         \
 141         } else {                                                        \
 142                 /*                                                      \
 143                  * Mark the irq stack inuse _before_ and unmark _after_ \
 144                  * switching stacks. Interrupts are disabled in both    \
 145                  * places. Invoke the stack switch macro with the call  \
 146                  * sequence which matches the above direct invocation.  \
 147                  */                                                     \
 148                 __this_cpu_write(hardirq_stack_inuse, true);            \
 149                 call_on_irqstack(func, asm_call, constr);               \
 150                 __this_cpu_write(hardirq_stack_inuse, false);           \
 151         }                                                               \
 152 }
 153
 154 /*
 155  * Function call sequence for __call_on_irqstack() for system vectors.
 156  *
 157  * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
 158  * mechanism because these functions are global and cannot be optimized out
 159  * when compiling a particular source file which uses one of these macros.
 160  *
 161  * The argument (regs) does not need to be pushed or stashed in a callee
 162  * saved register to be safe vs. the irq_enter_rcu() call because the
 163  * clobbers already prevent the compiler from storing it in a callee
 164  * clobbered register. As the compiler has to preserve @regs for the final
 165  * call to idtentry_exit() anyway, it's likely that it does not cause extra
 166  * effort for this asm magic.
 167  */
 168 #define ASM_CALL_SYSVEC                                                 \
 169         "call irq_enter_rcu                             \n"             \
 170         ASM_CALL_ARG1                                                   \
 171         "call irq_exit_rcu                              \n"
 172
 173 #define SYSVEC_CONSTRAINTS      , [arg1] "r" (regs)
 174
 175 #define run_sysvec_on_irqstack_cond(func, regs)                         \
 176 {                                                                       \
 177         assert_function_type(func, void (*)(struct pt_regs *));         \
 178         assert_arg_type(regs, struct pt_regs *);                        \
 179                                                                         \
 180         call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC,              \
 181                               SYSVEC_CONSTRAINTS, regs);                \
 182 }
 183
 184 /*
 185  * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store
 186  * @regs and @vector in callee saved registers.
 187  */
 188 #define ASM_CALL_IRQ                                                    \
 189         "call irq_enter_rcu                             \n"             \
 190         ASM_CALL_ARG2                                                   \
 191         "call irq_exit_rcu                              \n"
 192
 193 #define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" ((unsigned long)vector)
 194
 195 #define run_irq_on_irqstack_cond(func, regs, vector)                    \
 196 {                                                                       \
 197         assert_function_type(func, void (*)(struct pt_regs *, u32));    \
 198         assert_arg_type(regs, struct pt_regs *);                        \
 199         assert_arg_type(vector, u32);                                   \
 200                                                                         \
 201         call_on_irqstack_cond(func, regs, ASM_CALL_IRQ,                 \
 202                               IRQ_CONSTRAINTS, regs, vector);           \
 203 }
 204
 205 /*
 206  * Macro to invoke __do_softirq on the irq stack. This is only called from
 207  * task context when bottom halves are about to be reenabled and soft
 208  * interrupts are pending to be processed. The interrupt stack cannot be in
 209  * use here.
 210  */
 211 #define do_softirq_own_stack()                                          \
 212 {                                                                       \
 213         __this_cpu_write(hardirq_stack_inuse, true);                    \
 214         call_on_irqstack(__do_softirq, ASM_CALL_ARG0);                  \
 215         __this_cpu_write(hardirq_stack_inuse, false);                   \
 216 }
 217
 218 #else /* CONFIG_X86_64 */
 219 /* System vector handlers always run on the stack they interrupted. */
 220 #define run_sysvec_on_irqstack_cond(func, regs)                         \
 221 {                                                                       \
 222         irq_enter_rcu();                                                \
 223         func(regs);                                                     \
 224         irq_exit_rcu();                                                 \
 225 }
 226
 227 /* Switches to the irq stack within func() */
 228 #define run_irq_on_irqstack_cond(func, regs, vector)                    \
 229 {                                                                       \
 230         irq_enter_rcu();                                                \
 231         func(regs, vector);                                             \
 232         irq_exit_rcu();                                                 \
 233 }
 234
 235 #endif /* !CONFIG_X86_64 */
 236
 237 #endif