Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Sleepable Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright (C) IBM Corporation, 2006
6 * Copyright (C) Fujitsu, 2012
7 *
8 * Author: Paul McKenney <paulmck@linux.ibm.com>
9 * Lai Jiangshan <laijs@cn.fujitsu.com>
10 *
11 * For detailed explanation of Read-Copy Update mechanism see -
12 * Documentation/RCU/ *.txt
13 *
14 */
15
16#ifndef _LINUX_SRCU_H
17#define _LINUX_SRCU_H
18
19#include <linux/mutex.h>
20#include <linux/rcupdate.h>
21#include <linux/workqueue.h>
22#include <linux/rcu_segcblist.h>
23
24struct srcu_struct;
25
26#ifdef CONFIG_DEBUG_LOCK_ALLOC
27
28int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
29 struct lock_class_key *key);
30
31#define init_srcu_struct(ssp) \
32({ \
33 static struct lock_class_key __srcu_key; \
34 \
35 __init_srcu_struct((ssp), #ssp, &__srcu_key); \
36})
37
38#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
39#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
40
41int init_srcu_struct(struct srcu_struct *ssp);
42
43#define __SRCU_DEP_MAP_INIT(srcu_name)
44#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
45
46/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
47#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().
48#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().
49// 0x4 // SRCU-lite is no longer with us.
50#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().
51#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
52 SRCU_READ_FLAVOR_FAST) // All of the above.
53#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_FAST
54 // Flavors requiring synchronize_rcu()
55 // instead of smp_mb().
56void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
57
58#ifdef CONFIG_TINY_SRCU
59#include <linux/srcutiny.h>
60#elif defined(CONFIG_TREE_SRCU)
61#include <linux/srcutree.h>
62#else
63#error "Unknown SRCU implementation specified to kernel configuration"
64#endif
65
66void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
67 void (*func)(struct rcu_head *head));
68void cleanup_srcu_struct(struct srcu_struct *ssp);
69void synchronize_srcu(struct srcu_struct *ssp);
70
71#define SRCU_GET_STATE_COMPLETED 0x1
72
73/**
74 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
75 *
76 * Returns a value that poll_state_synchronize_srcu() will always treat
77 * as a cookie whose grace period has already completed.
78 */
79static inline unsigned long get_completed_synchronize_srcu(void)
80{
81 return SRCU_GET_STATE_COMPLETED;
82}
83
84unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
85unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
86bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
87
88// Maximum number of unsigned long values corresponding to
89// not-yet-completed SRCU grace periods.
90#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2
91
92/**
93 * same_state_synchronize_srcu - Are two old-state values identical?
94 * @oldstate1: First old-state value.
95 * @oldstate2: Second old-state value.
96 *
97 * The two old-state values must have been obtained from either
98 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
99 * get_completed_synchronize_srcu(). Returns @true if the two values are
100 * identical and @false otherwise. This allows structures whose lifetimes
101 * are tracked by old-state values to push these values to a list header,
102 * allowing those structures to be slightly smaller.
103 */
104static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
105{
106 return oldstate1 == oldstate2;
107}
108
109#ifdef CONFIG_NEED_SRCU_NMI_SAFE
110int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
111void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
112#else
113static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
114{
115 return __srcu_read_lock(ssp);
116}
117static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
118{
119 __srcu_read_unlock(ssp, idx);
120}
121#endif /* CONFIG_NEED_SRCU_NMI_SAFE */
122
123void srcu_init(void);
124
125#ifdef CONFIG_DEBUG_LOCK_ALLOC
126
127/**
128 * srcu_read_lock_held - might we be in SRCU read-side critical section?
129 * @ssp: The srcu_struct structure to check
130 *
131 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
132 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
133 * this assumes we are in an SRCU read-side critical section unless it can
134 * prove otherwise.
135 *
136 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
137 * and while lockdep is disabled.
138 *
139 * Note that SRCU is based on its own statemachine and it doesn't
140 * relies on normal RCU, it can be called from the CPU which
141 * is in the idle loop from an RCU point of view or offline.
142 */
143static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
144{
145 if (!debug_lockdep_rcu_enabled())
146 return 1;
147 return lock_is_held(&ssp->dep_map);
148}
149
150/*
151 * Annotations provide deadlock detection for SRCU.
152 *
153 * Similar to other lockdep annotations, except there is an additional
154 * srcu_lock_sync(), which is basically an empty *write*-side critical section,
155 * see lock_sync() for more information.
156 */
157
158/* Annotates a srcu_read_lock() */
159static inline void srcu_lock_acquire(struct lockdep_map *map)
160{
161 lock_map_acquire_read(map);
162}
163
164/* Annotates a srcu_read_lock() */
165static inline void srcu_lock_release(struct lockdep_map *map)
166{
167 lock_map_release(map);
168}
169
170/* Annotates a synchronize_srcu() */
171static inline void srcu_lock_sync(struct lockdep_map *map)
172{
173 lock_map_sync(map);
174}
175
176#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
177
178static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
179{
180 return 1;
181}
182
183#define srcu_lock_acquire(m) do { } while (0)
184#define srcu_lock_release(m) do { } while (0)
185#define srcu_lock_sync(m) do { } while (0)
186
187#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
188
189
190/**
191 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
192 * @p: the pointer to fetch and protect for later dereferencing
193 * @ssp: pointer to the srcu_struct, which is used to check that we
194 * really are in an SRCU read-side critical section.
195 * @c: condition to check for update-side use
196 *
197 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
198 * critical section will result in an RCU-lockdep splat, unless @c evaluates
199 * to 1. The @c argument will normally be a logical expression containing
200 * lockdep_is_held() calls.
201 */
202#define srcu_dereference_check(p, ssp, c) \
203 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
204 (c) || srcu_read_lock_held(ssp), __rcu)
205
206/**
207 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
208 * @p: the pointer to fetch and protect for later dereferencing
209 * @ssp: pointer to the srcu_struct, which is used to check that we
210 * really are in an SRCU read-side critical section.
211 *
212 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
213 * is enabled, invoking this outside of an RCU read-side critical
214 * section will result in an RCU-lockdep splat.
215 */
216#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
217
218/**
219 * srcu_dereference_notrace - no tracing and no lockdep calls from here
220 * @p: the pointer to fetch and protect for later dereferencing
221 * @ssp: pointer to the srcu_struct, which is used to check that we
222 * really are in an SRCU read-side critical section.
223 */
224#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
225
226/**
227 * srcu_read_lock - register a new reader for an SRCU-protected structure.
228 * @ssp: srcu_struct in which to register the new reader.
229 *
230 * Enter an SRCU read-side critical section. Note that SRCU read-side
231 * critical sections may be nested. However, it is illegal to
232 * call anything that waits on an SRCU grace period for the same
233 * srcu_struct, whether directly or indirectly. Please note that
234 * one way to indirectly wait on an SRCU grace period is to acquire
235 * a mutex that is held elsewhere while calling synchronize_srcu() or
236 * synchronize_srcu_expedited().
237 *
238 * The return value from srcu_read_lock() is guaranteed to be
239 * non-negative. This value must be passed unaltered to the matching
240 * srcu_read_unlock(). Note that srcu_read_lock() and the matching
241 * srcu_read_unlock() must occur in the same context, for example, it is
242 * illegal to invoke srcu_read_unlock() in an irq handler if the matching
243 * srcu_read_lock() was invoked in process context. Or, for that matter to
244 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
245 * from another.
246 */
247static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
248{
249 int retval;
250
251 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
252 retval = __srcu_read_lock(ssp);
253 srcu_lock_acquire(&ssp->dep_map);
254 return retval;
255}
256
257/**
258 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
259 * @ssp: srcu_struct in which to register the new reader.
260 *
261 * Enter an SRCU read-side critical section, but for a light-weight
262 * smp_mb()-free reader. See srcu_read_lock() for more information.
263 *
264 * If srcu_read_lock_fast() is ever used on an srcu_struct structure,
265 * then none of the other flavors may be used, whether before, during,
266 * or after. Note that grace-period auto-expediting is disabled for _fast
267 * srcu_struct structures because auto-expedited grace periods invoke
268 * synchronize_rcu_expedited(), IPIs and all.
269 *
270 * Note that srcu_read_lock_fast() can be invoked only from those contexts
271 * where RCU is watching, that is, from contexts where it would be legal
272 * to invoke rcu_read_lock(). Otherwise, lockdep will complain.
273 */
274static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)
275{
276 struct srcu_ctr __percpu *retval;
277
278 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");
279 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
280 retval = __srcu_read_lock_fast(ssp);
281 rcu_try_lock_acquire(&ssp->dep_map);
282 return retval;
283}
284
285/*
286 * Used by tracing, cannot be traced and cannot call lockdep.
287 * See srcu_read_lock_fast() for more information.
288 */
289static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp)
290 __acquires(ssp)
291{
292 struct srcu_ctr __percpu *retval;
293
294 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
295 retval = __srcu_read_lock_fast(ssp);
296 return retval;
297}
298
299/**
300 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
301 * @ssp: srcu_struct in which to register the new reader.
302 *
303 * Enter a semaphore-like SRCU read-side critical section, but for
304 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and
305 * srcu_down_read() for more information.
306 *
307 * The same srcu_struct may be used concurrently by srcu_down_read_fast()
308 * and srcu_read_lock_fast().
309 */
310static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)
311{
312 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
313 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");
314 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
315 return __srcu_read_lock_fast(ssp);
316}
317
318/**
319 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
320 * @ssp: srcu_struct in which to register the new reader.
321 *
322 * Enter an SRCU read-side critical section, but in an NMI-safe manner.
323 * See srcu_read_lock() for more information.
324 *
325 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
326 * then none of the other flavors may be used, whether before, during,
327 * or after.
328 */
329static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
330{
331 int retval;
332
333 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
334 retval = __srcu_read_lock_nmisafe(ssp);
335 rcu_try_lock_acquire(&ssp->dep_map);
336 return retval;
337}
338
339/* Used by tracing, cannot be traced and cannot invoke lockdep. */
340static inline notrace int
341srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
342{
343 int retval;
344
345 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
346 retval = __srcu_read_lock(ssp);
347 return retval;
348}
349
350/**
351 * srcu_down_read - register a new reader for an SRCU-protected structure.
352 * @ssp: srcu_struct in which to register the new reader.
353 *
354 * Enter a semaphore-like SRCU read-side critical section. Note that
355 * SRCU read-side critical sections may be nested. However, it is
356 * illegal to call anything that waits on an SRCU grace period for the
357 * same srcu_struct, whether directly or indirectly. Please note that
358 * one way to indirectly wait on an SRCU grace period is to acquire
359 * a mutex that is held elsewhere while calling synchronize_srcu() or
360 * synchronize_srcu_expedited(). But if you want lockdep to help you
361 * keep this stuff straight, you should instead use srcu_read_lock().
362 *
363 * The semaphore-like nature of srcu_down_read() means that the matching
364 * srcu_up_read() can be invoked from some other context, for example,
365 * from some other task or from an irq handler. However, neither
366 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
367 *
368 * Calls to srcu_down_read() may be nested, similar to the manner in
369 * which calls to down_read() may be nested. The same srcu_struct may be
370 * used concurrently by srcu_down_read() and srcu_read_lock().
371 */
372static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
373{
374 WARN_ON_ONCE(in_nmi());
375 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
376 return __srcu_read_lock(ssp);
377}
378
379/**
380 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
381 * @ssp: srcu_struct in which to unregister the old reader.
382 * @idx: return value from corresponding srcu_read_lock().
383 *
384 * Exit an SRCU read-side critical section.
385 */
386static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
387 __releases(ssp)
388{
389 WARN_ON_ONCE(idx & ~0x1);
390 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
391 srcu_lock_release(&ssp->dep_map);
392 __srcu_read_unlock(ssp, idx);
393}
394
395/**
396 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
397 * @ssp: srcu_struct in which to unregister the old reader.
398 * @scp: return value from corresponding srcu_read_lock_fast().
399 *
400 * Exit a light-weight SRCU read-side critical section.
401 */
402static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
403 __releases(ssp)
404{
405 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
406 srcu_lock_release(&ssp->dep_map);
407 __srcu_read_unlock_fast(ssp, scp);
408 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");
409}
410
411/*
412 * Used by tracing, cannot be traced and cannot call lockdep.
413 * See srcu_read_unlock_fast() for more information.
414 */
415static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp,
416 struct srcu_ctr __percpu *scp) __releases(ssp)
417{
418 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
419 __srcu_read_unlock_fast(ssp, scp);
420}
421
422/**
423 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
424 * @ssp: srcu_struct in which to unregister the old reader.
425 * @scp: return value from corresponding srcu_read_lock_fast().
426 *
427 * Exit an SRCU read-side critical section, but not necessarily from
428 * the same context as the maching srcu_down_read_fast().
429 */
430static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
431 __releases(ssp)
432{
433 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
434 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
435 __srcu_read_unlock_fast(ssp, scp);
436 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast().");
437}
438
439/**
440 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
441 * @ssp: srcu_struct in which to unregister the old reader.
442 * @idx: return value from corresponding srcu_read_lock_nmisafe().
443 *
444 * Exit an SRCU read-side critical section, but in an NMI-safe manner.
445 */
446static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
447 __releases(ssp)
448{
449 WARN_ON_ONCE(idx & ~0x1);
450 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
451 rcu_lock_release(&ssp->dep_map);
452 __srcu_read_unlock_nmisafe(ssp, idx);
453}
454
455/* Used by tracing, cannot be traced and cannot call lockdep. */
456static inline notrace void
457srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
458{
459 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
460 __srcu_read_unlock(ssp, idx);
461}
462
463/**
464 * srcu_up_read - unregister a old reader from an SRCU-protected structure.
465 * @ssp: srcu_struct in which to unregister the old reader.
466 * @idx: return value from corresponding srcu_read_lock().
467 *
468 * Exit an SRCU read-side critical section, but not necessarily from
469 * the same context as the maching srcu_down_read().
470 */
471static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
472 __releases(ssp)
473{
474 WARN_ON_ONCE(idx & ~0x1);
475 WARN_ON_ONCE(in_nmi());
476 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
477 __srcu_read_unlock(ssp, idx);
478}
479
480/**
481 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
482 *
483 * Converts the preceding srcu_read_unlock into a two-way memory barrier.
484 *
485 * Call this after srcu_read_unlock, to guarantee that all memory operations
486 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
487 * the preceding srcu_read_unlock.
488 */
489static inline void smp_mb__after_srcu_read_unlock(void)
490{
491 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
492}
493
494/**
495 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
496 *
497 * Converts the preceding srcu_read_lock into a two-way memory barrier.
498 *
499 * Call this after srcu_read_lock, to guarantee that all memory operations
500 * that occur after smp_mb__after_srcu_read_lock will appear to happen after
501 * the preceding srcu_read_lock.
502 */
503static inline void smp_mb__after_srcu_read_lock(void)
504{
505 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */
506}
507
508DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
509 _T->idx = srcu_read_lock(_T->lock),
510 srcu_read_unlock(_T->lock, _T->idx),
511 int idx)
512
513DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct,
514 _T->scp = srcu_read_lock_fast(_T->lock),
515 srcu_read_unlock_fast(_T->lock, _T->scp),
516 struct srcu_ctr __percpu *scp)
517
518DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,
519 _T->scp = srcu_read_lock_fast_notrace(_T->lock),
520 srcu_read_unlock_fast_notrace(_T->lock, _T->scp),
521 struct srcu_ctr __percpu *scp)
522
523#endif