tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc/qspinlock: powerpc qspinlock implementation
Add a powerpc specific implementation of queued spinlocks. This is the
build framework with a very simple (non-queued) spinlock implementation
to begin with. Later changes add queueing, and other features and
optimisations one-at-a-time. It is done this way to more easily see how
the queued spinlocks are built, and to make performance and correctness
bisects more useful.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
[mpe: Drop paravirt.h & processor.h changes to fix 32-bit build]
[mpe: Fix 32-bit build of qspinlock.o & disallow GENERIC_LOCKBREAK per Nick]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/CONLLQB6DCJU.2ZPOS7T6S5GRR@bobo
authored by
Nicholas Piggin
and committed by
Michael Ellerman
9f61521c
247f34f7
+71
-75
+1
-2
arch/powerpc/Kconfig
+1
-2
arch/powerpc/Kconfig
···
96
96
config GENERIC_LOCKBREAK
97
97
bool
98
98
default y
99
-
depends on SMP && PREEMPTION
99
+
depends on SMP && PREEMPTION && !PPC_QUEUED_SPINLOCKS
100
100
101
101
config GENERIC_HWEIGHT
102
102
bool
···
154
154
select ARCH_USE_CMPXCHG_LOCKREF if PPC64
155
155
select ARCH_USE_MEMTEST
156
156
select ARCH_USE_QUEUED_RWLOCKS if PPC_QUEUED_SPINLOCKS
157
-
select ARCH_USE_QUEUED_SPINLOCKS if PPC_QUEUED_SPINLOCKS
158
157
select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
159
158
select ARCH_WANT_IPC_PARSE_VERSION
160
159
select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
+35
-63
arch/powerpc/include/asm/qspinlock.h
+35
-63
arch/powerpc/include/asm/qspinlock.h
···
2
2
#ifndef _ASM_POWERPC_QSPINLOCK_H
3
3
#define _ASM_POWERPC_QSPINLOCK_H
4
4
5
-
#include <asm-generic/qspinlock_types.h>
5
+
#include <linux/atomic.h>
6
+
#include <linux/compiler.h>
7
+
#include <asm/qspinlock_types.h>
6
8
#include <asm/paravirt.h>
7
9
8
-
#define _Q_PENDING_LOOPS (1 << 9) /* not tuned */
9
-
10
-
#ifdef CONFIG_PARAVIRT_SPINLOCKS
11
-
extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
12
-
extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
13
-
extern void __pv_queued_spin_unlock(struct qspinlock *lock);
14
-
15
-
static __always_inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
10
+
static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
16
11
{
17
-
if (!is_shared_processor())
18
-
native_queued_spin_lock_slowpath(lock, val);
19
-
else
20
-
__pv_queued_spin_lock_slowpath(lock, val);
12
+
return atomic_read(&lock->val);
21
13
}
22
14
23
-
#define queued_spin_unlock queued_spin_unlock
24
-
static inline void queued_spin_unlock(struct qspinlock *lock)
15
+
static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
25
16
{
26
-
if (!is_shared_processor())
27
-
smp_store_release(&lock->locked, 0);
28
-
else
29
-
__pv_queued_spin_unlock(lock);
17
+
return !atomic_read(&lock.val);
30
18
}
31
19
32
-
#else
33
-
extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
34
-
#endif
20
+
static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
21
+
{
22
+
return 0;
23
+
}
24
+
25
+
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
26
+
{
27
+
return atomic_cmpxchg_acquire(&lock->val, 0, 1) == 0;
28
+
}
29
+
30
+
void queued_spin_lock_slowpath(struct qspinlock *lock);
35
31
36
32
static __always_inline void queued_spin_lock(struct qspinlock *lock)
37
33
{
38
-
u32 val = 0;
39
-
40
-
if (likely(arch_atomic_try_cmpxchg_lock(&lock->val, &val, _Q_LOCKED_VAL)))
41
-
return;
42
-
43
-
queued_spin_lock_slowpath(lock, val);
34
+
if (!queued_spin_trylock(lock))
35
+
queued_spin_lock_slowpath(lock);
44
36
}
45
-
#define queued_spin_lock queued_spin_lock
37
+
38
+
static inline void queued_spin_unlock(struct qspinlock *lock)
39
+
{
40
+
atomic_set_release(&lock->val, 0);
41
+
}
42
+
43
+
#define arch_spin_is_locked(l) queued_spin_is_locked(l)
44
+
#define arch_spin_is_contended(l) queued_spin_is_contended(l)
45
+
#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l)
46
+
#define arch_spin_lock(l) queued_spin_lock(l)
47
+
#define arch_spin_trylock(l) queued_spin_trylock(l)
48
+
#define arch_spin_unlock(l) queued_spin_unlock(l)
46
49
47
50
#ifdef CONFIG_PARAVIRT_SPINLOCKS
48
-
#define SPIN_THRESHOLD (1<<15) /* not tuned */
49
-
50
-
static __always_inline void pv_wait(u8 *ptr, u8 val)
51
-
{
52
-
if (*ptr != val)
53
-
return;
54
-
yield_to_any();
55
-
/*
56
-
* We could pass in a CPU here if waiting in the queue and yield to
57
-
* the previous CPU in the queue.
58
-
*/
59
-
}
60
-
61
-
static __always_inline void pv_kick(int cpu)
62
-
{
63
-
prod_cpu(cpu);
64
-
}
65
-
66
-
extern void __pv_init_lock_hash(void);
67
-
68
-
static inline void pv_spinlocks_init(void)
69
-
{
70
-
__pv_init_lock_hash();
71
-
}
72
-
51
+
void pv_spinlocks_init(void);
52
+
#else
53
+
static inline void pv_spinlocks_init(void) { }
73
54
#endif
74
-
75
-
/*
76
-
* Queued spinlocks rely heavily on smp_cond_load_relaxed() to busy-wait,
77
-
* which was found to have performance problems if implemented with
78
-
* the preferred spin_begin()/spin_end() SMT priority pattern. Use the
79
-
* generic version instead.
80
-
*/
81
-
82
-
#include <asm-generic/qspinlock.h>
83
55
84
56
#endif /* _ASM_POWERPC_QSPINLOCK_H */
-7
arch/powerpc/include/asm/qspinlock_paravirt.h
-7
arch/powerpc/include/asm/qspinlock_paravirt.h
+13
arch/powerpc/include/asm/qspinlock_types.h
+13
arch/powerpc/include/asm/qspinlock_types.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
#ifndef _ASM_POWERPC_QSPINLOCK_TYPES_H
3
+
#define _ASM_POWERPC_QSPINLOCK_TYPES_H
4
+
5
+
#include <linux/types.h>
6
+
7
+
typedef struct qspinlock {
8
+
atomic_t val;
9
+
} arch_spinlock_t;
10
+
11
+
#define __ARCH_SPIN_LOCK_UNLOCKED { .val = ATOMIC_INIT(0) }
12
+
13
+
#endif /* _ASM_POWERPC_QSPINLOCK_TYPES_H */
+1
-1
arch/powerpc/include/asm/spinlock.h
+1
-1
arch/powerpc/include/asm/spinlock.h
+1
-1
arch/powerpc/include/asm/spinlock_types.h
+1
-1
arch/powerpc/include/asm/spinlock_types.h
+3
-1
arch/powerpc/lib/Makefile
+3
-1
arch/powerpc/lib/Makefile
+17
arch/powerpc/lib/qspinlock.c
+17
arch/powerpc/lib/qspinlock.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
#include <linux/export.h>
3
+
#include <linux/processor.h>
4
+
#include <asm/qspinlock.h>
5
+
6
+
void queued_spin_lock_slowpath(struct qspinlock *lock)
7
+
{
8
+
while (!queued_spin_trylock(lock))
9
+
cpu_relax();
10
+
}
11
+
EXPORT_SYMBOL(queued_spin_lock_slowpath);
12
+
13
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
14
+
void pv_spinlocks_init(void)
15
+
{
16
+
}
17
+
#endif