tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
seqlock: livelock fix
Thomas Gleixner debugged a particularly ugly seqlock related livelock:
do not process the seq-read section if we know it beforehand that the
test at the end of the section will fail ...
Signed-off-by: Ingo Molnar <mingo@elte.hu>
+29
-17
+29
-17
include/linux/seqlock.h
+29
-17
include/linux/seqlock.h
···
85
85
/* Start of read calculation -- fetch last complete writer token */
86
86
static __always_inline unsigned read_seqbegin(const seqlock_t *sl)
87
87
{
88
-
unsigned ret = sl->sequence;
88
+
unsigned ret;
89
+
90
+
repeat:
91
+
ret = sl->sequence;
89
92
smp_rmb();
93
+
if (unlikely(ret & 1)) {
94
+
cpu_relax();
95
+
goto repeat;
96
+
}
97
+
90
98
return ret;
91
99
}
92
100
93
-
/* Test if reader processed invalid data.
94
-
* If initial values is odd,
95
-
* then writer had already started when section was entered
96
-
* If sequence value changed
97
-
* then writer changed data while in section
98
-
*
99
-
* Using xor saves one conditional branch.
101
+
/*
102
+
* Test if reader processed invalid data.
103
+
*
104
+
* If sequence value changed then writer changed data while in section.
100
105
*/
101
-
static __always_inline int read_seqretry(const seqlock_t *sl, unsigned iv)
106
+
static __always_inline int read_seqretry(const seqlock_t *sl, unsigned start)
102
107
{
103
108
smp_rmb();
104
-
return (iv & 1) | (sl->sequence ^ iv);
109
+
110
+
return (sl->sequence != start);
105
111
}
106
112
107
113
···
128
122
/* Start of read using pointer to a sequence counter only. */
129
123
static inline unsigned read_seqcount_begin(const seqcount_t *s)
130
124
{
131
-
unsigned ret = s->sequence;
125
+
unsigned ret;
126
+
127
+
repeat:
128
+
ret = s->sequence;
132
129
smp_rmb();
130
+
if (unlikely(ret & 1)) {
131
+
cpu_relax();
132
+
goto repeat;
133
+
}
133
134
return ret;
134
135
}
135
136
136
-
/* Test if reader processed invalid data.
137
-
* Equivalent to: iv is odd or sequence number has changed.
138
-
* (iv & 1) || (*s != iv)
139
-
* Using xor saves one conditional branch.
137
+
/*
138
+
* Test if reader processed invalid data because sequence number has changed.
140
139
*/
141
-
static inline int read_seqcount_retry(const seqcount_t *s, unsigned iv)
140
+
static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
142
141
{
143
142
smp_rmb();
144
-
return (iv & 1) | (s->sequence ^ iv);
143
+
144
+
return s->sequence != start;
145
145
}
146
146
147
147