tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_QSPINLOCK_PARAVIRT_H
3#define __ASM_QSPINLOCK_PARAVIRT_H
4
5/*
6 * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
7 * registers. For i386, however, only 1 32-bit register needs to be saved
8 * and restored. So an optimized version of __pv_queued_spin_unlock() is
9 * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
10 */
11#ifdef CONFIG_64BIT
12
13PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
14#define __pv_queued_spin_unlock __pv_queued_spin_unlock
15#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
16#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
17
18/*
19 * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
20 * which combines the registers saving trunk and the body of the following
21 * C code:
22 *
23 * void __pv_queued_spin_unlock(struct qspinlock *lock)
24 * {
25 * struct __qspinlock *l = (void *)lock;
26 * u8 lockval = cmpxchg(&l->locked, _Q_LOCKED_VAL, 0);
27 *
28 * if (likely(lockval == _Q_LOCKED_VAL))
29 * return;
30 * pv_queued_spin_unlock_slowpath(lock, lockval);
31 * }
32 *
33 * For x86-64,
34 * rdi = lock (first argument)
35 * rsi = lockval (second argument)
36 * rdx = internal variable (set to 0)
37 */
38asm (".pushsection .text;"
39 ".globl " PV_UNLOCK ";"
40 ".type " PV_UNLOCK ", @function;"
41 ".align 4,0x90;"
42 PV_UNLOCK ": "
43 FRAME_BEGIN
44 "push %rdx;"
45 "mov $0x1,%eax;"
46 "xor %edx,%edx;"
47 "lock cmpxchg %dl,(%rdi);"
48 "cmp $0x1,%al;"
49 "jne .slowpath;"
50 "pop %rdx;"
51 FRAME_END
52 "ret;"
53 ".slowpath: "
54 "push %rsi;"
55 "movzbl %al,%esi;"
56 "call " PV_UNLOCK_SLOWPATH ";"
57 "pop %rsi;"
58 "pop %rdx;"
59 FRAME_END
60 "ret;"
61 ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
62 ".popsection");
63
64#else /* CONFIG_64BIT */
65
66extern void __pv_queued_spin_unlock(struct qspinlock *lock);
67PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
68
69#endif /* CONFIG_64BIT */
70#endif