tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking/atomic: add generic arch_*() bitops
Now that all architectures provide arch_atomic_long_*(), we can
implement the generic bitops atop these rather than atop
atomic_long_*(), and provide arch_*() forms of the bitops that are safe
to use in noinstr code.
Now that all architectures provide arch_atomic_long_*(), we can
build the generic arch_*() bitops atop these, which can be safely used
in noinstr code. The regular bitop wrappers are built atop these.
As the generic non-atomic bitops use plain accesses, these will be
implicitly instrumented unless they are inlined into noinstr functions
(which is similar to arch_atomic*_read() when based on READ_ONCE()).
The wrappers are modified so that where the underlying arch_*() function
uses a plain access, no explicit instrumentation is added, as this is
redundant and could result in confusing reports.
Since function prototypes get excessively long with both an `arch_`
prefix and `__always_inline` attribute, the return type and function
attributes have been split onto a separate line, matching the style of
the generated atomic headers.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210713105253.7615-6-mark.rutland@arm.com
authored by
Mark Rutland
and committed by
Peter Zijlstra
cf3ee3c8
67d1b0de
+83
-50
+20
-12
include/asm-generic/bitops/atomic.h
+20
-12
include/asm-generic/bitops/atomic.h
···
11
11
* See Documentation/atomic_bitops.txt for details.
12
12
*/
13
13
14
-
static __always_inline void set_bit(unsigned int nr, volatile unsigned long *p)
14
+
static __always_inline void
15
+
arch_set_bit(unsigned int nr, volatile unsigned long *p)
15
16
{
16
17
p += BIT_WORD(nr);
17
-
atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);
18
+
arch_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);
18
19
}
19
20
20
-
static __always_inline void clear_bit(unsigned int nr, volatile unsigned long *p)
21
+
static __always_inline void
22
+
arch_clear_bit(unsigned int nr, volatile unsigned long *p)
21
23
{
22
24
p += BIT_WORD(nr);
23
-
atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);
25
+
arch_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);
24
26
}
25
27
26
-
static __always_inline void change_bit(unsigned int nr, volatile unsigned long *p)
28
+
static __always_inline void
29
+
arch_change_bit(unsigned int nr, volatile unsigned long *p)
27
30
{
28
31
p += BIT_WORD(nr);
29
-
atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);
32
+
arch_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);
30
33
}
31
34
32
-
static inline int test_and_set_bit(unsigned int nr, volatile unsigned long *p)
35
+
static __always_inline int
36
+
arch_test_and_set_bit(unsigned int nr, volatile unsigned long *p)
33
37
{
34
38
long old;
35
39
unsigned long mask = BIT_MASK(nr);
···
42
38
if (READ_ONCE(*p) & mask)
43
39
return 1;
44
40
45
-
old = atomic_long_fetch_or(mask, (atomic_long_t *)p);
41
+
old = arch_atomic_long_fetch_or(mask, (atomic_long_t *)p);
46
42
return !!(old & mask);
47
43
}
48
44
49
-
static inline int test_and_clear_bit(unsigned int nr, volatile unsigned long *p)
45
+
static __always_inline int
46
+
arch_test_and_clear_bit(unsigned int nr, volatile unsigned long *p)
50
47
{
51
48
long old;
52
49
unsigned long mask = BIT_MASK(nr);
···
56
51
if (!(READ_ONCE(*p) & mask))
57
52
return 0;
58
53
59
-
old = atomic_long_fetch_andnot(mask, (atomic_long_t *)p);
54
+
old = arch_atomic_long_fetch_andnot(mask, (atomic_long_t *)p);
60
55
return !!(old & mask);
61
56
}
62
57
63
-
static inline int test_and_change_bit(unsigned int nr, volatile unsigned long *p)
58
+
static __always_inline int
59
+
arch_test_and_change_bit(unsigned int nr, volatile unsigned long *p)
64
60
{
65
61
long old;
66
62
unsigned long mask = BIT_MASK(nr);
67
63
68
64
p += BIT_WORD(nr);
69
-
old = atomic_long_fetch_xor(mask, (atomic_long_t *)p);
65
+
old = arch_atomic_long_fetch_xor(mask, (atomic_long_t *)p);
70
66
return !!(old & mask);
71
67
}
68
+
69
+
#include <asm-generic/bitops/instrumented-atomic.h>
72
70
73
71
#endif /* _ASM_GENERIC_BITOPS_ATOMIC_H */
+14
-7
include/asm-generic/bitops/instrumented-non-atomic.h
+14
-7
include/asm-generic/bitops/instrumented-non-atomic.h
···
24
24
*/
25
25
static inline void __set_bit(long nr, volatile unsigned long *addr)
26
26
{
27
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
27
+
if (!__is_defined(arch___set_bit_uses_plain_access))
28
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
28
29
arch___set_bit(nr, addr);
29
30
}
30
31
···
40
39
*/
41
40
static inline void __clear_bit(long nr, volatile unsigned long *addr)
42
41
{
43
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
42
+
if (!__is_defined(arch___clear_bit_uses_plain_access))
43
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
44
44
arch___clear_bit(nr, addr);
45
45
}
46
46
···
56
54
*/
57
55
static inline void __change_bit(long nr, volatile unsigned long *addr)
58
56
{
59
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
57
+
if (!__is_defined(arch___change_bit_uses_plain_access))
58
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
60
59
arch___change_bit(nr, addr);
61
60
}
62
61
···
95
92
*/
96
93
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
97
94
{
98
-
__instrument_read_write_bitop(nr, addr);
95
+
if (!__is_defined(arch___test_and_set_bit_uses_plain_access))
96
+
__instrument_read_write_bitop(nr, addr);
99
97
return arch___test_and_set_bit(nr, addr);
100
98
}
101
99
···
110
106
*/
111
107
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
112
108
{
113
-
__instrument_read_write_bitop(nr, addr);
109
+
if (!__is_defined(arch___test_and_clear_bit_uses_plain_access))
110
+
__instrument_read_write_bitop(nr, addr);
114
111
return arch___test_and_clear_bit(nr, addr);
115
112
}
116
113
···
125
120
*/
126
121
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
127
122
{
128
-
__instrument_read_write_bitop(nr, addr);
123
+
if (!__is_defined(arch___test_and_change_bit_uses_plain_access))
124
+
__instrument_read_write_bitop(nr, addr);
129
125
return arch___test_and_change_bit(nr, addr);
130
126
}
131
127
···
137
131
*/
138
132
static inline bool test_bit(long nr, const volatile unsigned long *addr)
139
133
{
140
-
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
134
+
if (!__is_defined(arch_test_bit_uses_plain_access))
135
+
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
141
136
return arch_test_bit(nr, addr);
142
137
}
143
138
+21
-18
include/asm-generic/bitops/lock.h
+21
-18
include/asm-generic/bitops/lock.h
···
7
7
#include <asm/barrier.h>
8
8
9
9
/**
10
-
* test_and_set_bit_lock - Set a bit and return its old value, for lock
10
+
* arch_test_and_set_bit_lock - Set a bit and return its old value, for lock
11
11
* @nr: Bit to set
12
12
* @addr: Address to count from
13
13
*
···
15
15
* the returned value is 0.
16
16
* It can be used to implement bit locks.
17
17
*/
18
-
static inline int test_and_set_bit_lock(unsigned int nr,
19
-
volatile unsigned long *p)
18
+
static __always_inline int
19
+
arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p)
20
20
{
21
21
long old;
22
22
unsigned long mask = BIT_MASK(nr);
···
25
25
if (READ_ONCE(*p) & mask)
26
26
return 1;
27
27
28
-
old = atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
28
+
old = arch_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
29
29
return !!(old & mask);
30
30
}
31
31
32
32
33
33
/**
34
-
* clear_bit_unlock - Clear a bit in memory, for unlock
34
+
* arch_clear_bit_unlock - Clear a bit in memory, for unlock
35
35
* @nr: the bit to set
36
36
* @addr: the address to start counting from
37
37
*
38
38
* This operation is atomic and provides release barrier semantics.
39
39
*/
40
-
static inline void clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
40
+
static __always_inline void
41
+
arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
41
42
{
42
43
p += BIT_WORD(nr);
43
-
atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
44
+
arch_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
44
45
}
45
46
46
47
/**
47
-
* __clear_bit_unlock - Clear a bit in memory, for unlock
48
+
* arch___clear_bit_unlock - Clear a bit in memory, for unlock
48
49
* @nr: the bit to set
49
50
* @addr: the address to start counting from
50
51
*
···
55
54
*
56
55
* See for example x86's implementation.
57
56
*/
58
-
static inline void __clear_bit_unlock(unsigned int nr,
59
-
volatile unsigned long *p)
57
+
static inline void
58
+
arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
60
59
{
61
60
unsigned long old;
62
61
63
62
p += BIT_WORD(nr);
64
63
old = READ_ONCE(*p);
65
64
old &= ~BIT_MASK(nr);
66
-
atomic_long_set_release((atomic_long_t *)p, old);
65
+
arch_atomic_long_set_release((atomic_long_t *)p, old);
67
66
}
68
67
69
68
/**
70
-
* clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
71
-
* byte is negative, for unlock.
69
+
* arch_clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
70
+
* byte is negative, for unlock.
72
71
* @nr: the bit to clear
73
72
* @addr: the address to start counting from
74
73
*
75
74
* This is a bit of a one-trick-pony for the filemap code, which clears
76
75
* PG_locked and tests PG_waiters,
77
76
*/
78
-
#ifndef clear_bit_unlock_is_negative_byte
79
-
static inline bool clear_bit_unlock_is_negative_byte(unsigned int nr,
80
-
volatile unsigned long *p)
77
+
#ifndef arch_clear_bit_unlock_is_negative_byte
78
+
static inline bool arch_clear_bit_unlock_is_negative_byte(unsigned int nr,
79
+
volatile unsigned long *p)
81
80
{
82
81
long old;
83
82
unsigned long mask = BIT_MASK(nr);
84
83
85
84
p += BIT_WORD(nr);
86
-
old = atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
85
+
old = arch_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
87
86
return !!(old & BIT(7));
88
87
}
89
-
#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte
88
+
#define arch_clear_bit_unlock_is_negative_byte arch_clear_bit_unlock_is_negative_byte
90
89
#endif
90
+
91
+
#include <asm-generic/bitops/instrumented-lock.h>
91
92
92
93
#endif /* _ASM_GENERIC_BITOPS_LOCK_H_ */
+28
-13
include/asm-generic/bitops/non-atomic.h
+28
-13
include/asm-generic/bitops/non-atomic.h
···
5
5
#include <asm/types.h>
6
6
7
7
/**
8
-
* __set_bit - Set a bit in memory
8
+
* arch___set_bit - Set a bit in memory
9
9
* @nr: the bit to set
10
10
* @addr: the address to start counting from
11
11
*
···
13
13
* If it's called on the same region of memory simultaneously, the effect
14
14
* may be that only one operation succeeds.
15
15
*/
16
-
static inline void __set_bit(int nr, volatile unsigned long *addr)
16
+
static __always_inline void
17
+
arch___set_bit(int nr, volatile unsigned long *addr)
17
18
{
18
19
unsigned long mask = BIT_MASK(nr);
19
20
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
20
21
21
22
*p |= mask;
22
23
}
24
+
#define arch___set_bit_uses_plain_access
23
25
24
-
static inline void __clear_bit(int nr, volatile unsigned long *addr)
26
+
static __always_inline void
27
+
arch___clear_bit(int nr, volatile unsigned long *addr)
25
28
{
26
29
unsigned long mask = BIT_MASK(nr);
27
30
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
28
31
29
32
*p &= ~mask;
30
33
}
34
+
#define arch___clear_bit_uses_plain_access
31
35
32
36
/**
33
-
* __change_bit - Toggle a bit in memory
37
+
* arch___change_bit - Toggle a bit in memory
34
38
* @nr: the bit to change
35
39
* @addr: the address to start counting from
36
40
*
···
42
38
* If it's called on the same region of memory simultaneously, the effect
43
39
* may be that only one operation succeeds.
44
40
*/
45
-
static inline void __change_bit(int nr, volatile unsigned long *addr)
41
+
static __always_inline
42
+
void arch___change_bit(int nr, volatile unsigned long *addr)
46
43
{
47
44
unsigned long mask = BIT_MASK(nr);
48
45
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
49
46
50
47
*p ^= mask;
51
48
}
49
+
#define arch___change_bit_uses_plain_access
52
50
53
51
/**
54
-
* __test_and_set_bit - Set a bit and return its old value
52
+
* arch___test_and_set_bit - Set a bit and return its old value
55
53
* @nr: Bit to set
56
54
* @addr: Address to count from
57
55
*
···
61
55
* If two examples of this operation race, one can appear to succeed
62
56
* but actually fail. You must protect multiple accesses with a lock.
63
57
*/
64
-
static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
58
+
static __always_inline int
59
+
arch___test_and_set_bit(int nr, volatile unsigned long *addr)
65
60
{
66
61
unsigned long mask = BIT_MASK(nr);
67
62
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
···
71
64
*p = old | mask;
72
65
return (old & mask) != 0;
73
66
}
67
+
#define arch___test_and_set_bit_uses_plain_access
74
68
75
69
/**
76
-
* __test_and_clear_bit - Clear a bit and return its old value
70
+
* arch___test_and_clear_bit - Clear a bit and return its old value
77
71
* @nr: Bit to clear
78
72
* @addr: Address to count from
79
73
*
···
82
74
* If two examples of this operation race, one can appear to succeed
83
75
* but actually fail. You must protect multiple accesses with a lock.
84
76
*/
85
-
static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
77
+
static __always_inline int
78
+
arch___test_and_clear_bit(int nr, volatile unsigned long *addr)
86
79
{
87
80
unsigned long mask = BIT_MASK(nr);
88
81
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
···
92
83
*p = old & ~mask;
93
84
return (old & mask) != 0;
94
85
}
86
+
#define arch___test_and_clear_bit_uses_plain_access
95
87
96
88
/* WARNING: non atomic and it can be reordered! */
97
-
static inline int __test_and_change_bit(int nr,
98
-
volatile unsigned long *addr)
89
+
static __always_inline int
90
+
arch___test_and_change_bit(int nr, volatile unsigned long *addr)
99
91
{
100
92
unsigned long mask = BIT_MASK(nr);
101
93
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
···
105
95
*p = old ^ mask;
106
96
return (old & mask) != 0;
107
97
}
98
+
#define arch___test_and_change_bit_uses_plain_access
108
99
109
100
/**
110
-
* test_bit - Determine whether a bit is set
101
+
* arch_test_bit - Determine whether a bit is set
111
102
* @nr: bit number to test
112
103
* @addr: Address to start counting from
113
104
*/
114
-
static inline int test_bit(int nr, const volatile unsigned long *addr)
105
+
static __always_inline int
106
+
arch_test_bit(int nr, const volatile unsigned long *addr)
115
107
{
116
108
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
117
109
}
110
+
#define arch_test_bit_uses_plain_access
111
+
112
+
#include <asm-generic/bitops/instrumented-non-atomic.h>
118
113
119
114
#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */