tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_RQSPINLOCK_H
3#define _ASM_RQSPINLOCK_H
4
5#include <asm/barrier.h>
6
7/*
8 * Hardcode res_smp_cond_load_acquire implementations for arm64 to a custom
9 * version based on [0]. In rqspinlock code, our conditional expression involves
10 * checking the value _and_ additionally a timeout. However, on arm64, the
11 * WFE-based implementation may never spin again if no stores occur to the
12 * locked byte in the lock word. As such, we may be stuck forever if
13 * event-stream based unblocking is not available on the platform for WFE spin
14 * loops (arch_timer_evtstrm_available).
15 *
16 * Once support for smp_cond_load_acquire_timewait [0] lands, we can drop this
17 * copy-paste.
18 *
19 * While we rely on the implementation to amortize the cost of sampling
20 * cond_expr for us, it will not happen when event stream support is
21 * unavailable, time_expr check is amortized. This is not the common case, and
22 * it would be difficult to fit our logic in the time_expr_ns >= time_limit_ns
23 * comparison, hence just let it be. In case of event-stream, the loop is woken
24 * up at microsecond granularity.
25 *
26 * [0]: https://lore.kernel.org/lkml/20250203214911.898276-1-ankur.a.arora@oracle.com
27 */
28
29#ifndef smp_cond_load_acquire_timewait
30
31#define smp_cond_time_check_count 200
32
33#define __smp_cond_load_relaxed_spinwait(ptr, cond_expr, time_expr_ns, \
34 time_limit_ns) ({ \
35 typeof(ptr) __PTR = (ptr); \
36 __unqual_scalar_typeof(*ptr) VAL; \
37 unsigned int __count = 0; \
38 for (;;) { \
39 VAL = READ_ONCE(*__PTR); \
40 if (cond_expr) \
41 break; \
42 cpu_relax(); \
43 if (__count++ < smp_cond_time_check_count) \
44 continue; \
45 if ((time_expr_ns) >= (time_limit_ns)) \
46 break; \
47 __count = 0; \
48 } \
49 (typeof(*ptr))VAL; \
50})
51
52#define __smp_cond_load_acquire_timewait(ptr, cond_expr, \
53 time_expr_ns, time_limit_ns) \
54({ \
55 typeof(ptr) __PTR = (ptr); \
56 __unqual_scalar_typeof(*ptr) VAL; \
57 for (;;) { \
58 VAL = smp_load_acquire(__PTR); \
59 if (cond_expr) \
60 break; \
61 __cmpwait_relaxed(__PTR, VAL); \
62 if ((time_expr_ns) >= (time_limit_ns)) \
63 break; \
64 } \
65 (typeof(*ptr))VAL; \
66})
67
68#define smp_cond_load_acquire_timewait(ptr, cond_expr, \
69 time_expr_ns, time_limit_ns) \
70({ \
71 __unqual_scalar_typeof(*ptr) _val; \
72 int __wfe = arch_timer_evtstrm_available(); \
73 \
74 if (likely(__wfe)) { \
75 _val = __smp_cond_load_acquire_timewait(ptr, cond_expr, \
76 time_expr_ns, \
77 time_limit_ns); \
78 } else { \
79 _val = __smp_cond_load_relaxed_spinwait(ptr, cond_expr, \
80 time_expr_ns, \
81 time_limit_ns); \
82 smp_acquire__after_ctrl_dep(); \
83 } \
84 (typeof(*ptr))_val; \
85})
86
87#endif
88
89#define res_smp_cond_load_acquire(v, c) smp_cond_load_acquire_timewait(v, c, 0, 1)
90
91#include <asm-generic/rqspinlock.h>
92
93#endif /* _ASM_RQSPINLOCK_H */