x86/include/asm/qspinlock_paravirt.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_QSPINLOCK_PARAVIRT_H
   3 #define __ASM_QSPINLOCK_PARAVIRT_H
   4
   5 #include <asm/ibt.h>
   6
   7 /*
   8  * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
   9  * registers. For i386, however, only 1 32-bit register needs to be saved
  10  * and restored. So an optimized version of __pv_queued_spin_unlock() is
  11  * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
  12  */
  13 #ifdef CONFIG_64BIT
  14
  15 __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
  16 #define __pv_queued_spin_unlock __pv_queued_spin_unlock
  17 #define PV_UNLOCK               "__raw_callee_save___pv_queued_spin_unlock"
  18 #define PV_UNLOCK_SLOWPATH      "__raw_callee_save___pv_queued_spin_unlock_slowpath"
  19
  20 /*
  21  * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
  22  * which combines the registers saving trunk and the body of the following
  23  * C code.  Note that it puts the code in the .spinlock.text section which
  24  * is equivalent to adding __lockfunc in the C code:
  25  *
  26  * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
  27  * {
  28  *      u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
  29  *
  30  *      if (likely(lockval == _Q_LOCKED_VAL))
  31  *              return;
  32  *      pv_queued_spin_unlock_slowpath(lock, lockval);
  33  * }
  34  *
  35  * For x86-64,
  36  *   rdi = lock              (first argument)
  37  *   rsi = lockval           (second argument)
  38  *   rdx = internal variable (set to 0)
  39  */
  40 asm    (".pushsection .spinlock.text, \"ax\";"
  41         ".globl " PV_UNLOCK ";"
  42         ".type " PV_UNLOCK ", @function;"
  43         ".align 4,0x90;"
  44         PV_UNLOCK ": "
  45         ASM_ENDBR
  46         FRAME_BEGIN
  47         "push  %rdx;"
  48         "mov   $0x1,%eax;"
  49         "xor   %edx,%edx;"
  50         LOCK_PREFIX "cmpxchg %dl,(%rdi);"
  51         "cmp   $0x1,%al;"
  52         "jne   .slowpath;"
  53         "pop   %rdx;"
  54         FRAME_END
  55         ASM_RET
  56         ".slowpath: "
  57         "push   %rsi;"
  58         "movzbl %al,%esi;"
  59         "call " PV_UNLOCK_SLOWPATH ";"
  60         "pop    %rsi;"
  61         "pop    %rdx;"
  62         FRAME_END
  63         ASM_RET
  64         ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
  65         ".popsection");
  66
  67 #else /* CONFIG_64BIT */
  68
  69 extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
  70 __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
  71
  72 #endif /* CONFIG_64BIT */
  73 #endif