Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * Out of line spinlock code.
3 *
4 * Copyright IBM Corp. 2004, 2006
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
6 */
7
8#include <linux/types.h>
9#include <linux/export.h>
10#include <linux/spinlock.h>
11#include <linux/init.h>
12#include <linux/smp.h>
13#include <asm/io.h>
14
15int spin_retry = -1;
16
17static int __init spin_retry_init(void)
18{
19 if (spin_retry < 0)
20 spin_retry = 1000;
21 return 0;
22}
23early_initcall(spin_retry_init);
24
25/**
26 * spin_retry= parameter
27 */
28static int __init spin_retry_setup(char *str)
29{
30 spin_retry = simple_strtoul(str, &str, 0);
31 return 1;
32}
33__setup("spin_retry=", spin_retry_setup);
34
35void arch_spin_lock_wait(arch_spinlock_t *lp)
36{
37 int cpu = SPINLOCK_LOCKVAL;
38 int owner, count, first_diag;
39
40 first_diag = 1;
41 while (1) {
42 owner = ACCESS_ONCE(lp->lock);
43 /* Try to get the lock if it is free. */
44 if (!owner) {
45 if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
46 return;
47 continue;
48 }
49 /* First iteration: check if the lock owner is running. */
50 if (first_diag && arch_vcpu_is_preempted(~owner)) {
51 smp_yield_cpu(~owner);
52 first_diag = 0;
53 continue;
54 }
55 /* Loop for a while on the lock value. */
56 count = spin_retry;
57 do {
58 owner = ACCESS_ONCE(lp->lock);
59 } while (owner && count-- > 0);
60 if (!owner)
61 continue;
62 /*
63 * For multiple layers of hypervisors, e.g. z/VM + LPAR
64 * yield the CPU unconditionally. For LPAR rely on the
65 * sense running status.
66 */
67 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) {
68 smp_yield_cpu(~owner);
69 first_diag = 0;
70 }
71 }
72}
73EXPORT_SYMBOL(arch_spin_lock_wait);
74
75void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
76{
77 int cpu = SPINLOCK_LOCKVAL;
78 int owner, count, first_diag;
79
80 local_irq_restore(flags);
81 first_diag = 1;
82 while (1) {
83 owner = ACCESS_ONCE(lp->lock);
84 /* Try to get the lock if it is free. */
85 if (!owner) {
86 local_irq_disable();
87 if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
88 return;
89 local_irq_restore(flags);
90 continue;
91 }
92 /* Check if the lock owner is running. */
93 if (first_diag && arch_vcpu_is_preempted(~owner)) {
94 smp_yield_cpu(~owner);
95 first_diag = 0;
96 continue;
97 }
98 /* Loop for a while on the lock value. */
99 count = spin_retry;
100 do {
101 owner = ACCESS_ONCE(lp->lock);
102 } while (owner && count-- > 0);
103 if (!owner)
104 continue;
105 /*
106 * For multiple layers of hypervisors, e.g. z/VM + LPAR
107 * yield the CPU unconditionally. For LPAR rely on the
108 * sense running status.
109 */
110 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) {
111 smp_yield_cpu(~owner);
112 first_diag = 0;
113 }
114 }
115}
116EXPORT_SYMBOL(arch_spin_lock_wait_flags);
117
118int arch_spin_trylock_retry(arch_spinlock_t *lp)
119{
120 int cpu = SPINLOCK_LOCKVAL;
121 int owner, count;
122
123 for (count = spin_retry; count > 0; count--) {
124 owner = READ_ONCE(lp->lock);
125 /* Try to get the lock if it is free. */
126 if (!owner) {
127 if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
128 return 1;
129 }
130 }
131 return 0;
132}
133EXPORT_SYMBOL(arch_spin_trylock_retry);
134
135void _raw_read_lock_wait(arch_rwlock_t *rw)
136{
137 int count = spin_retry;
138 int owner, old;
139
140#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
141 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
142#endif
143 owner = 0;
144 while (1) {
145 if (count-- <= 0) {
146 if (owner && arch_vcpu_is_preempted(~owner))
147 smp_yield_cpu(~owner);
148 count = spin_retry;
149 }
150 old = ACCESS_ONCE(rw->lock);
151 owner = ACCESS_ONCE(rw->owner);
152 if (old < 0)
153 continue;
154 if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
155 return;
156 }
157}
158EXPORT_SYMBOL(_raw_read_lock_wait);
159
160int _raw_read_trylock_retry(arch_rwlock_t *rw)
161{
162 int count = spin_retry;
163 int old;
164
165 while (count-- > 0) {
166 old = ACCESS_ONCE(rw->lock);
167 if (old < 0)
168 continue;
169 if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
170 return 1;
171 }
172 return 0;
173}
174EXPORT_SYMBOL(_raw_read_trylock_retry);
175
176#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
177
178void _raw_write_lock_wait(arch_rwlock_t *rw, int prev)
179{
180 int count = spin_retry;
181 int owner, old;
182
183 owner = 0;
184 while (1) {
185 if (count-- <= 0) {
186 if (owner && arch_vcpu_is_preempted(~owner))
187 smp_yield_cpu(~owner);
188 count = spin_retry;
189 }
190 old = ACCESS_ONCE(rw->lock);
191 owner = ACCESS_ONCE(rw->owner);
192 smp_mb();
193 if (old >= 0) {
194 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
195 old = prev;
196 }
197 if ((old & 0x7fffffff) == 0 && prev >= 0)
198 break;
199 }
200}
201EXPORT_SYMBOL(_raw_write_lock_wait);
202
203#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
204
205void _raw_write_lock_wait(arch_rwlock_t *rw)
206{
207 int count = spin_retry;
208 int owner, old, prev;
209
210 prev = 0x80000000;
211 owner = 0;
212 while (1) {
213 if (count-- <= 0) {
214 if (owner && arch_vcpu_is_preempted(~owner))
215 smp_yield_cpu(~owner);
216 count = spin_retry;
217 }
218 old = ACCESS_ONCE(rw->lock);
219 owner = ACCESS_ONCE(rw->owner);
220 if (old >= 0 &&
221 __atomic_cmpxchg_bool(&rw->lock, old, old | 0x80000000))
222 prev = old;
223 else
224 smp_mb();
225 if ((old & 0x7fffffff) == 0 && prev >= 0)
226 break;
227 }
228}
229EXPORT_SYMBOL(_raw_write_lock_wait);
230
231#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
232
233int _raw_write_trylock_retry(arch_rwlock_t *rw)
234{
235 int count = spin_retry;
236 int old;
237
238 while (count-- > 0) {
239 old = ACCESS_ONCE(rw->lock);
240 if (old)
241 continue;
242 if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000))
243 return 1;
244 }
245 return 0;
246}
247EXPORT_SYMBOL(_raw_write_trylock_retry);
248
249void arch_lock_relax(int cpu)
250{
251 if (!cpu)
252 return;
253 if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(~cpu))
254 return;
255 smp_yield_cpu(~cpu);
256}
257EXPORT_SYMBOL(arch_lock_relax);