arch/x86/include/asm/switch_to.h

   1 #ifndef _ASM_X86_SWITCH_TO_H
   2 #define _ASM_X86_SWITCH_TO_H
   3
   4 #include <asm/nospec-branch.h>
   5
   6 struct task_struct; /* one of the stranger aspects of C forward declarations */
   7 __visible struct task_struct *__switch_to(struct task_struct *prev,
   8                                            struct task_struct *next);
   9
  10 #ifdef CONFIG_X86_32
  11
  12 #ifdef CONFIG_CC_STACKPROTECTOR
  13 #define __switch_canary                                                 \
  14         "movl %P[task_canary](%[next]), %%ebx\n\t"                      \
  15         "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
  16 #define __switch_canary_oparam                                          \
  17         , [stack_canary] "=m" (stack_canary.canary)
  18 #define __switch_canary_iparam                                          \
  19         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
  20 #else   /* CC_STACKPROTECTOR */
  21 #define __switch_canary
  22 #define __switch_canary_oparam
  23 #define __switch_canary_iparam
  24 #endif  /* CC_STACKPROTECTOR */
  25
  26 #ifdef CONFIG_RETPOLINE
  27         /*
  28          * When switching from a shallower to a deeper call stack
  29          * the RSB may either underflow or use entries populated
  30          * with userspace addresses. On CPUs where those concerns
  31          * exist, overwrite the RSB with entries which capture
  32          * speculative execution to prevent attack.
  33          */
  34 #define __retpoline_fill_return_buffer                                  \
  35         ALTERNATIVE("jmp 910f",                                         \
  36                 __stringify(__FILL_RETURN_BUFFER(%%ebx, RSB_CLEAR_LOOPS, %%esp)),\
  37                 X86_FEATURE_RSB_CTXSW)                                  \
  38         "910:\n\t"
  39 #else
  40 #define __retpoline_fill_return_buffer
  41 #endif
  42
  43 /*
  44  * Saving eflags is important. It switches not only IOPL between tasks,
  45  * it also protects other tasks from NT leaking through sysenter etc.
  46  */
  47 #define switch_to(prev, next, last)                                     \
  48 do {                                                                    \
  49         /*                                                              \
  50          * Context-switching clobbers all registers, so we clobber      \
  51          * them explicitly, via unused output variables.                \
  52          * (EAX and EBP is not listed because EBP is saved/restored     \
  53          * explicitly for wchan access and EAX is the return value of   \
  54          * __switch_to())                                               \
  55          */                                                             \
  56         unsigned long ebx, ecx, edx, esi, edi;                          \
  57                                                                         \
  58         asm volatile("pushfl\n\t"               /* save    flags */     \
  59                      "pushl %%ebp\n\t"          /* save    EBP   */     \
  60                      "movl %%esp,%[prev_sp]\n\t"        /* save    ESP   */ \
  61                      "movl %[next_sp],%%esp\n\t"        /* restore ESP   */ \
  62                      "movl $1f,%[prev_ip]\n\t"  /* save    EIP   */     \
  63                      "pushl %[next_ip]\n\t"     /* restore EIP   */     \
  64                      __switch_canary                                    \
  65                      __retpoline_fill_return_buffer                     \
  66                      "jmp __switch_to\n"        /* regparm call  */     \
  67                      "1:\t"                                             \
  68                      "popl %%ebp\n\t"           /* restore EBP   */     \
  69                      "popfl\n"                  /* restore flags */     \
  70                                                                         \
  71                      /* output parameters */                            \
  72                      : [prev_sp] "=m" (prev->thread.sp),                \
  73                        [prev_ip] "=m" (prev->thread.ip),                \
  74                        "=a" (last),                                     \
  75                                                                         \
  76                        /* clobbered output registers: */                \
  77                        "=b" (ebx), "=c" (ecx), "=d" (edx),              \
  78                        "=S" (esi), "=D" (edi)                           \
  79                                                                         \
  80                        __switch_canary_oparam                           \
  81                                                                         \
  82                        /* input parameters: */                          \
  83                      : [next_sp]  "m" (next->thread.sp),                \
  84                        [next_ip]  "m" (next->thread.ip),                \
  85                                                                         \
  86                        /* regparm parameters for __switch_to(): */      \
  87                        [prev]     "a" (prev),                           \
  88                        [next]     "d" (next)                            \
  89                                                                         \
  90                        __switch_canary_iparam                           \
  91                                                                         \
  92                      : /* reloaded segment registers */                 \
  93                         "memory");                                      \
  94 } while (0)
  95
  96 #else /* CONFIG_X86_32 */
  97
  98 /* frame pointer must be last for get_wchan */
  99 #define SAVE_CONTEXT    "pushq %%rbp ; movq %%rsi,%%rbp\n\t"
 100 #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp\t"
 101
 102 #define __EXTRA_CLOBBER  \
 103         , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
 104           "r12", "r13", "r14", "r15", "flags"
 105
 106 #ifdef CONFIG_CC_STACKPROTECTOR
 107 #define __switch_canary                                                   \
 108         "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
 109         "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
 110 #define __switch_canary_oparam                                            \
 111         , [gs_canary] "=m" (irq_stack_union.stack_canary)
 112 #define __switch_canary_iparam                                            \
 113         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
 114 #else   /* CC_STACKPROTECTOR */
 115 #define __switch_canary
 116 #define __switch_canary_oparam
 117 #define __switch_canary_iparam
 118 #endif  /* CC_STACKPROTECTOR */
 119
 120 #ifdef CONFIG_RETPOLINE
 121         /*
 122          * When switching from a shallower to a deeper call stack
 123          * the RSB may either underflow or use entries populated
 124          * with userspace addresses. On CPUs where those concerns
 125          * exist, overwrite the RSB with entries which capture
 126          * speculative execution to prevent attack.
 127          */
 128 #define __retpoline_fill_return_buffer                                  \
 129         ALTERNATIVE("jmp 910f",                                         \
 130                 __stringify(__FILL_RETURN_BUFFER(%%r12, RSB_CLEAR_LOOPS, %%rsp)),\
 131                 X86_FEATURE_RSB_CTXSW)                                  \
 132         "910:\n\t"
 133 #else
 134 #define __retpoline_fill_return_buffer
 135 #endif
 136
 137 /*
 138  * There is no need to save or restore flags, because flags are always
 139  * clean in kernel mode, with the possible exception of IOPL.  Kernel IOPL
 140  * has no effect.
 141  */
 142 #define switch_to(prev, next, last) \
 143         asm volatile(SAVE_CONTEXT                                         \
 144              "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */       \
 145              "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */    \
 146              "call __switch_to\n\t"                                       \
 147              "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
 148              __switch_canary                                              \
 149              __retpoline_fill_return_buffer                               \
 150              "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
 151              "movq %%rax,%%rdi\n\t"                                       \
 152              "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"                 \
 153              "jnz   ret_from_fork\n\t"                                    \
 154              RESTORE_CONTEXT                                              \
 155              : "=a" (last)                                                \
 156                __switch_canary_oparam                                     \
 157              : [next] "S" (next), [prev] "D" (prev),                      \
 158                [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
 159                [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
 160                [_tif_fork] "i" (_TIF_FORK),                               \
 161                [thread_info] "i" (offsetof(struct task_struct, stack)),   \
 162                [current_task] "m" (current_task)                          \
 163                __switch_canary_iparam                                     \
 164              : "memory", "cc" __EXTRA_CLOBBER)
 165
 166 #endif /* CONFIG_X86_32 */
 167
 168 #endif /* _ASM_X86_SWITCH_TO_H */