arch/x86/include/asm/qspinlock_paravirt.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_QSPINLOCK_PARAVIRT_H
   3 #define __ASM_QSPINLOCK_PARAVIRT_H
   4
   5 /*
   6  * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
   7  * registers. For i386, however, only 1 32-bit register needs to be saved
   8  * and restored. So an optimized version of __pv_queued_spin_unlock() is
   9  * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
  10  */
  11 #ifdef CONFIG_64BIT
  12
  13 PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  14 #define __pv_queued_spin_unlock __pv_queued_spin_unlock
  15 #define PV_UNLOCK               "__raw_callee_save___pv_queued_spin_unlock"
  16 #define PV_UNLOCK_SLOWPATH      "__raw_callee_save___pv_queued_spin_unlock_slowpath"
  17
  18 /*
  19  * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
  20  * which combines the registers saving trunk and the body of the following
  21  * C code:
  22  *
  23  * void __pv_queued_spin_unlock(struct qspinlock *lock)
  24  * {
  25  *      struct __qspinlock *l = (void *)lock;
  26  *      u8 lockval = cmpxchg(&l->locked, _Q_LOCKED_VAL, 0);
  27  *
  28  *      if (likely(lockval == _Q_LOCKED_VAL))
  29  *              return;
  30  *      pv_queued_spin_unlock_slowpath(lock, lockval);
  31  * }
  32  *
  33  * For x86-64,
  34  *   rdi = lock              (first argument)
  35  *   rsi = lockval           (second argument)
  36  *   rdx = internal variable (set to 0)
  37  */
  38 asm    (".pushsection .text;"
  39         ".globl " PV_UNLOCK ";"
  40         ".type " PV_UNLOCK ", @function;"
  41         ".align 4,0x90;"
  42         PV_UNLOCK ": "
  43         FRAME_BEGIN
  44         "push  %rdx;"
  45         "mov   $0x1,%eax;"
  46         "xor   %edx,%edx;"
  47         "lock cmpxchg %dl,(%rdi);"
  48         "cmp   $0x1,%al;"
  49         "jne   .slowpath;"
  50         "pop   %rdx;"
  51         FRAME_END
  52         "ret;"
  53         ".slowpath: "
  54         "push   %rsi;"
  55         "movzbl %al,%esi;"
  56         "call " PV_UNLOCK_SLOWPATH ";"
  57         "pop    %rsi;"
  58         "pop    %rdx;"
  59         FRAME_END
  60         "ret;"
  61         ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
  62         ".popsection");
  63
  64 #else /* CONFIG_64BIT */
  65
  66 extern void __pv_queued_spin_unlock(struct qspinlock *lock);
  67 PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
  68
  69 #endif /* CONFIG_64BIT */
  70 #endif