tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
s390/spinlock: add niai spinlock hints
The z14 machine introduces new mode of the next-instruction-access-intent
NIAI instruction. With NIAI-8 it is possible to pin a cache-line on a
CPU for a small amount of time, NIAI-7 releases the cache-line again.
Finally NIAI-4 can be used to prevent the CPU to speculatively access
memory beyond the compare-and-swap instruction to get the lock.
Use these instruction in the spinlock code.
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
+56
-40
+5
-4
arch/s390/include/asm/spinlock.h
+5
-4
arch/s390/include/asm/spinlock.h
···
92
92
{
93
93
typecheck(int, lp->lock);
94
94
asm volatile(
95
-
"st %1,%0\n"
96
-
: "+Q" (lp->lock)
97
-
: "d" (0)
98
-
: "cc", "memory");
95
+
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
96
+
" .long 0xb2fa0070\n" /* NIAI 7 */
97
+
#endif
98
+
" st %1,%0\n"
99
+
: "=Q" (lp->lock) : "d" (0) : "cc", "memory");
99
100
}
100
101
101
102
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
+51
-36
arch/s390/lib/spinlock.c
+51
-36
arch/s390/lib/spinlock.c
···
32
32
}
33
33
__setup("spin_retry=", spin_retry_setup);
34
34
35
+
static inline int arch_load_niai4(int *lock)
36
+
{
37
+
int owner;
38
+
39
+
asm volatile(
40
+
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
41
+
" .long 0xb2fa0040\n" /* NIAI 4 */
42
+
#endif
43
+
" l %0,%1\n"
44
+
: "=d" (owner) : "Q" (*lock) : "memory");
45
+
return owner;
46
+
}
47
+
48
+
static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
49
+
{
50
+
int expected = old;
51
+
52
+
asm volatile(
53
+
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
54
+
" .long 0xb2fa0080\n" /* NIAI 8 */
55
+
#endif
56
+
" cs %0,%3,%1\n"
57
+
: "=d" (old), "=Q" (*lock)
58
+
: "0" (old), "d" (new), "Q" (*lock)
59
+
: "cc", "memory");
60
+
return expected == old;
61
+
}
62
+
35
63
void arch_spin_lock_wait(arch_spinlock_t *lp)
36
64
{
37
65
int cpu = SPINLOCK_LOCKVAL;
38
-
int owner, count, first_diag;
66
+
int owner, count;
39
67
40
-
first_diag = 1;
68
+
/* Pass the virtual CPU to the lock holder if it is not running */
69
+
owner = arch_load_niai4(&lp->lock);
70
+
if (owner && arch_vcpu_is_preempted(~owner))
71
+
smp_yield_cpu(~owner);
72
+
73
+
count = spin_retry;
41
74
while (1) {
42
-
owner = ACCESS_ONCE(lp->lock);
75
+
owner = arch_load_niai4(&lp->lock);
43
76
/* Try to get the lock if it is free. */
44
77
if (!owner) {
45
-
if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
78
+
if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
46
79
return;
47
80
continue;
48
81
}
49
-
/* First iteration: check if the lock owner is running. */
50
-
if (first_diag && arch_vcpu_is_preempted(~owner)) {
51
-
smp_yield_cpu(~owner);
52
-
first_diag = 0;
82
+
if (count-- >= 0)
53
83
continue;
54
-
}
55
-
/* Loop for a while on the lock value. */
56
84
count = spin_retry;
57
-
do {
58
-
owner = ACCESS_ONCE(lp->lock);
59
-
} while (owner && count-- > 0);
60
-
if (!owner)
61
-
continue;
62
85
/*
63
86
* For multiple layers of hypervisors, e.g. z/VM + LPAR
64
87
* yield the CPU unconditionally. For LPAR rely on the
65
88
* sense running status.
66
89
*/
67
-
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) {
90
+
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
68
91
smp_yield_cpu(~owner);
69
-
first_diag = 0;
70
-
}
71
92
}
72
93
}
73
94
EXPORT_SYMBOL(arch_spin_lock_wait);
···
96
75
void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
97
76
{
98
77
int cpu = SPINLOCK_LOCKVAL;
99
-
int owner, count, first_diag;
78
+
int owner, count;
100
79
101
80
local_irq_restore(flags);
102
-
first_diag = 1;
81
+
82
+
/* Pass the virtual CPU to the lock holder if it is not running */
83
+
owner = arch_load_niai4(&lp->lock);
84
+
if (owner && arch_vcpu_is_preempted(~owner))
85
+
smp_yield_cpu(~owner);
86
+
87
+
count = spin_retry;
103
88
while (1) {
104
-
owner = ACCESS_ONCE(lp->lock);
89
+
owner = arch_load_niai4(&lp->lock);
105
90
/* Try to get the lock if it is free. */
106
91
if (!owner) {
107
92
local_irq_disable();
108
-
if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
93
+
if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
109
94
return;
110
95
local_irq_restore(flags);
111
96
continue;
112
97
}
113
-
/* Check if the lock owner is running. */
114
-
if (first_diag && arch_vcpu_is_preempted(~owner)) {
115
-
smp_yield_cpu(~owner);
116
-
first_diag = 0;
98
+
if (count-- >= 0)
117
99
continue;
118
-
}
119
-
/* Loop for a while on the lock value. */
120
100
count = spin_retry;
121
-
do {
122
-
owner = ACCESS_ONCE(lp->lock);
123
-
} while (owner && count-- > 0);
124
-
if (!owner)
125
-
continue;
126
101
/*
127
102
* For multiple layers of hypervisors, e.g. z/VM + LPAR
128
103
* yield the CPU unconditionally. For LPAR rely on the
129
104
* sense running status.
130
105
*/
131
-
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) {
106
+
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
132
107
smp_yield_cpu(~owner);
133
-
first_diag = 0;
134
-
}
135
108
}
136
109
}
137
110
EXPORT_SYMBOL(arch_spin_lock_wait_flags);