tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking/atomic: simplify non-atomic wrappers
Since the non-atomic arch_*() bitops use plain accesses, they are
implicitly instrumnted by the compiler, and we work around this in the
instrumented wrappers to avoid double instrumentation.
It's simpler to avoid the wrappers entirely, and use the preprocessor to
alias the arch_*() bitops to their regular versions, removing the need
for checks in the instrumented wrappers.
Suggested-by: Marco Elver <elver@google.com>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Marco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20210721155813.17082-1-mark.rutland@arm.com
authored by
Mark Rutland
and committed by
Peter Zijlstra
9248e52f
cf3ee3c8
+14
-23
+7
-14
include/asm-generic/bitops/instrumented-non-atomic.h
+7
-14
include/asm-generic/bitops/instrumented-non-atomic.h
···
24
24
*/
25
25
static inline void __set_bit(long nr, volatile unsigned long *addr)
26
26
{
27
-
if (!__is_defined(arch___set_bit_uses_plain_access))
28
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
27
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
29
28
arch___set_bit(nr, addr);
30
29
}
31
30
···
39
40
*/
40
41
static inline void __clear_bit(long nr, volatile unsigned long *addr)
41
42
{
42
-
if (!__is_defined(arch___clear_bit_uses_plain_access))
43
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
43
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
44
44
arch___clear_bit(nr, addr);
45
45
}
46
46
···
54
56
*/
55
57
static inline void __change_bit(long nr, volatile unsigned long *addr)
56
58
{
57
-
if (!__is_defined(arch___change_bit_uses_plain_access))
58
-
instrument_write(addr + BIT_WORD(nr), sizeof(long));
59
+
instrument_write(addr + BIT_WORD(nr), sizeof(long));
59
60
arch___change_bit(nr, addr);
60
61
}
61
62
···
92
95
*/
93
96
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
94
97
{
95
-
if (!__is_defined(arch___test_and_set_bit_uses_plain_access))
96
-
__instrument_read_write_bitop(nr, addr);
98
+
__instrument_read_write_bitop(nr, addr);
97
99
return arch___test_and_set_bit(nr, addr);
98
100
}
99
101
···
106
110
*/
107
111
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
108
112
{
109
-
if (!__is_defined(arch___test_and_clear_bit_uses_plain_access))
110
-
__instrument_read_write_bitop(nr, addr);
113
+
__instrument_read_write_bitop(nr, addr);
111
114
return arch___test_and_clear_bit(nr, addr);
112
115
}
113
116
···
120
125
*/
121
126
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
122
127
{
123
-
if (!__is_defined(arch___test_and_change_bit_uses_plain_access))
124
-
__instrument_read_write_bitop(nr, addr);
128
+
__instrument_read_write_bitop(nr, addr);
125
129
return arch___test_and_change_bit(nr, addr);
126
130
}
127
131
···
131
137
*/
132
138
static inline bool test_bit(long nr, const volatile unsigned long *addr)
133
139
{
134
-
if (!__is_defined(arch_test_bit_uses_plain_access))
135
-
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
140
+
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
136
141
return arch_test_bit(nr, addr);
137
142
}
138
143
+7
-9
include/asm-generic/bitops/non-atomic.h
+7
-9
include/asm-generic/bitops/non-atomic.h
···
21
21
22
22
*p |= mask;
23
23
}
24
-
#define arch___set_bit_uses_plain_access
24
+
#define __set_bit arch___set_bit
25
25
26
26
static __always_inline void
27
27
arch___clear_bit(int nr, volatile unsigned long *addr)
···
31
31
32
32
*p &= ~mask;
33
33
}
34
-
#define arch___clear_bit_uses_plain_access
34
+
#define __clear_bit arch___clear_bit
35
35
36
36
/**
37
37
* arch___change_bit - Toggle a bit in memory
···
50
50
51
51
*p ^= mask;
52
52
}
53
-
#define arch___change_bit_uses_plain_access
53
+
#define __change_bit arch___change_bit
54
54
55
55
/**
56
56
* arch___test_and_set_bit - Set a bit and return its old value
···
71
71
*p = old | mask;
72
72
return (old & mask) != 0;
73
73
}
74
-
#define arch___test_and_set_bit_uses_plain_access
74
+
#define __test_and_set_bit arch___test_and_set_bit
75
75
76
76
/**
77
77
* arch___test_and_clear_bit - Clear a bit and return its old value
···
92
92
*p = old & ~mask;
93
93
return (old & mask) != 0;
94
94
}
95
-
#define arch___test_and_clear_bit_uses_plain_access
95
+
#define __test_and_clear_bit arch___test_and_clear_bit
96
96
97
97
/* WARNING: non atomic and it can be reordered! */
98
98
static __always_inline int
···
105
105
*p = old ^ mask;
106
106
return (old & mask) != 0;
107
107
}
108
-
#define arch___test_and_change_bit_uses_plain_access
108
+
#define __test_and_change_bit arch___test_and_change_bit
109
109
110
110
/**
111
111
* arch_test_bit - Determine whether a bit is set
···
117
117
{
118
118
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
119
119
}
120
-
#define arch_test_bit_uses_plain_access
121
-
122
-
#include <asm-generic/bitops/instrumented-non-atomic.h>
120
+
#define test_bit arch_test_bit
123
121
124
122
#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */