Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Sleepable Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright (C) IBM Corporation, 2006
6 * Copyright (C) Fujitsu, 2012
7 *
8 * Author: Paul McKenney <paulmck@linux.ibm.com>
9 * Lai Jiangshan <laijs@cn.fujitsu.com>
10 *
11 * For detailed explanation of Read-Copy Update mechanism see -
12 * Documentation/RCU/ *.txt
13 *
14 */
15
16#ifndef _LINUX_SRCU_H
17#define _LINUX_SRCU_H
18
19#include <linux/mutex.h>
20#include <linux/rcupdate.h>
21#include <linux/workqueue.h>
22#include <linux/rcu_segcblist.h>
23
24struct srcu_struct;
25
26#ifdef CONFIG_DEBUG_LOCK_ALLOC
27
28int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
29#ifndef CONFIG_TINY_SRCU
30int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
31int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
32 struct lock_class_key *key);
33#endif // #ifndef CONFIG_TINY_SRCU
34
35#define init_srcu_struct(ssp) \
36({ \
37 static struct lock_class_key __srcu_key; \
38 \
39 __init_srcu_struct((ssp), #ssp, &__srcu_key); \
40})
41
42#define init_srcu_struct_fast(ssp) \
43({ \
44 static struct lock_class_key __srcu_key; \
45 \
46 __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \
47})
48
49#define init_srcu_struct_fast_updown(ssp) \
50({ \
51 static struct lock_class_key __srcu_key; \
52 \
53 __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \
54})
55
56#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
57#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
58
59int init_srcu_struct(struct srcu_struct *ssp);
60#ifndef CONFIG_TINY_SRCU
61int init_srcu_struct_fast(struct srcu_struct *ssp);
62int init_srcu_struct_fast_updown(struct srcu_struct *ssp);
63#endif // #ifndef CONFIG_TINY_SRCU
64
65#define __SRCU_DEP_MAP_INIT(srcu_name)
66#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
67
68/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
69#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().
70#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().
71// 0x4 // SRCU-lite is no longer with us.
72#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast().
73#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast().
74#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
75 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
76 // All of the above.
77#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
78 // Flavors requiring synchronize_rcu()
79 // instead of smp_mb().
80void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
81
82#ifdef CONFIG_TINY_SRCU
83#include <linux/srcutiny.h>
84#elif defined(CONFIG_TREE_SRCU)
85#include <linux/srcutree.h>
86#else
87#error "Unknown SRCU implementation specified to kernel configuration"
88#endif
89
90void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
91 void (*func)(struct rcu_head *head));
92void cleanup_srcu_struct(struct srcu_struct *ssp);
93void synchronize_srcu(struct srcu_struct *ssp);
94
95#define SRCU_GET_STATE_COMPLETED 0x1
96
97/**
98 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
99 *
100 * Returns a value that poll_state_synchronize_srcu() will always treat
101 * as a cookie whose grace period has already completed.
102 */
103static inline unsigned long get_completed_synchronize_srcu(void)
104{
105 return SRCU_GET_STATE_COMPLETED;
106}
107
108unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
109unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
110bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
111
112// Maximum number of unsigned long values corresponding to
113// not-yet-completed SRCU grace periods.
114#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2
115
116/**
117 * same_state_synchronize_srcu - Are two old-state values identical?
118 * @oldstate1: First old-state value.
119 * @oldstate2: Second old-state value.
120 *
121 * The two old-state values must have been obtained from either
122 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
123 * get_completed_synchronize_srcu(). Returns @true if the two values are
124 * identical and @false otherwise. This allows structures whose lifetimes
125 * are tracked by old-state values to push these values to a list header,
126 * allowing those structures to be slightly smaller.
127 */
128static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
129{
130 return oldstate1 == oldstate2;
131}
132
133#ifdef CONFIG_NEED_SRCU_NMI_SAFE
134int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
135void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
136#else
137static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
138{
139 return __srcu_read_lock(ssp);
140}
141static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
142{
143 __srcu_read_unlock(ssp, idx);
144}
145#endif /* CONFIG_NEED_SRCU_NMI_SAFE */
146
147void srcu_init(void);
148
149#ifdef CONFIG_DEBUG_LOCK_ALLOC
150
151/**
152 * srcu_read_lock_held - might we be in SRCU read-side critical section?
153 * @ssp: The srcu_struct structure to check
154 *
155 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
156 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
157 * this assumes we are in an SRCU read-side critical section unless it can
158 * prove otherwise.
159 *
160 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
161 * and while lockdep is disabled.
162 *
163 * Note that SRCU is based on its own statemachine and it doesn't
164 * relies on normal RCU, it can be called from the CPU which
165 * is in the idle loop from an RCU point of view or offline.
166 */
167static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
168{
169 if (!debug_lockdep_rcu_enabled())
170 return 1;
171 return lock_is_held(&ssp->dep_map);
172}
173
174/*
175 * Annotations provide deadlock detection for SRCU.
176 *
177 * Similar to other lockdep annotations, except there is an additional
178 * srcu_lock_sync(), which is basically an empty *write*-side critical section,
179 * see lock_sync() for more information.
180 */
181
182/* Annotates a srcu_read_lock() */
183static inline void srcu_lock_acquire(struct lockdep_map *map)
184{
185 lock_map_acquire_read(map);
186}
187
188/* Annotates a srcu_read_lock() */
189static inline void srcu_lock_release(struct lockdep_map *map)
190{
191 lock_map_release(map);
192}
193
194/* Annotates a synchronize_srcu() */
195static inline void srcu_lock_sync(struct lockdep_map *map)
196{
197 lock_map_sync(map);
198}
199
200#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
201
202static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
203{
204 return 1;
205}
206
207#define srcu_lock_acquire(m) do { } while (0)
208#define srcu_lock_release(m) do { } while (0)
209#define srcu_lock_sync(m) do { } while (0)
210
211#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
212
213
214/**
215 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
216 * @p: the pointer to fetch and protect for later dereferencing
217 * @ssp: pointer to the srcu_struct, which is used to check that we
218 * really are in an SRCU read-side critical section.
219 * @c: condition to check for update-side use
220 *
221 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
222 * critical section will result in an RCU-lockdep splat, unless @c evaluates
223 * to 1. The @c argument will normally be a logical expression containing
224 * lockdep_is_held() calls.
225 */
226#define srcu_dereference_check(p, ssp, c) \
227 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
228 (c) || srcu_read_lock_held(ssp), __rcu)
229
230/**
231 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
232 * @p: the pointer to fetch and protect for later dereferencing
233 * @ssp: pointer to the srcu_struct, which is used to check that we
234 * really are in an SRCU read-side critical section.
235 *
236 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
237 * is enabled, invoking this outside of an RCU read-side critical
238 * section will result in an RCU-lockdep splat.
239 */
240#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
241
242/**
243 * srcu_dereference_notrace - no tracing and no lockdep calls from here
244 * @p: the pointer to fetch and protect for later dereferencing
245 * @ssp: pointer to the srcu_struct, which is used to check that we
246 * really are in an SRCU read-side critical section.
247 */
248#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
249
250/**
251 * srcu_read_lock - register a new reader for an SRCU-protected structure.
252 * @ssp: srcu_struct in which to register the new reader.
253 *
254 * Enter an SRCU read-side critical section. Note that SRCU read-side
255 * critical sections may be nested. However, it is illegal to
256 * call anything that waits on an SRCU grace period for the same
257 * srcu_struct, whether directly or indirectly. Please note that
258 * one way to indirectly wait on an SRCU grace period is to acquire
259 * a mutex that is held elsewhere while calling synchronize_srcu() or
260 * synchronize_srcu_expedited().
261 *
262 * The return value from srcu_read_lock() is guaranteed to be
263 * non-negative. This value must be passed unaltered to the matching
264 * srcu_read_unlock(). Note that srcu_read_lock() and the matching
265 * srcu_read_unlock() must occur in the same context, for example, it is
266 * illegal to invoke srcu_read_unlock() in an irq handler if the matching
267 * srcu_read_lock() was invoked in process context. Or, for that matter to
268 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
269 * from another.
270 */
271static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
272{
273 int retval;
274
275 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
276 retval = __srcu_read_lock(ssp);
277 srcu_lock_acquire(&ssp->dep_map);
278 return retval;
279}
280
281/**
282 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
283 * @ssp: srcu_struct in which to register the new reader.
284 *
285 * Enter an SRCU read-side critical section, but for a light-weight
286 * smp_mb()-free reader. See srcu_read_lock() for more information. This
287 * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe().
288 *
289 * For srcu_read_lock_fast() to be used on an srcu_struct structure,
290 * that structure must have been defined using either DEFINE_SRCU_FAST()
291 * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with
292 * init_srcu_struct_fast() on the other. Such an srcu_struct structure
293 * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or
294 * srcu_{down,up}_read(). In kernels built with CONFIG_PROVE_RCU=y,
295 * __srcu_check_read_flavor() will complain bitterly if you ignore this
296 * restriction.
297 *
298 * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct
299 * structures because SRCU-fast expedited grace periods invoke
300 * synchronize_rcu_expedited(), IPIs and all. If you need expedited
301 * SRCU-fast grace periods, use synchronize_srcu_expedited().
302 *
303 * The srcu_read_lock_fast() function can be invoked only from those
304 * contexts where RCU is watching, that is, from contexts where it would
305 * be legal to invoke rcu_read_lock(). Otherwise, lockdep will complain.
306 */
307static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)
308{
309 struct srcu_ctr __percpu *retval;
310
311 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");
312 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
313 retval = __srcu_read_lock_fast(ssp);
314 rcu_try_lock_acquire(&ssp->dep_map);
315 return retval;
316}
317
318/**
319 * srcu_read_lock_fast_updown - register a new reader for an SRCU-fast-updown structure.
320 * @ssp: srcu_struct in which to register the new reader.
321 *
322 * Enter an SRCU read-side critical section, but for a light-weight
323 * smp_mb()-free reader. See srcu_read_lock() for more information.
324 * This function is compatible with srcu_down_read_fast(), but is not
325 * NMI-safe.
326 *
327 * For srcu_read_lock_fast_updown() to be used on an srcu_struct
328 * structure, that structure must have been defined using either
329 * DEFINE_SRCU_FAST_UPDOWN() or DEFINE_STATIC_SRCU_FAST_UPDOWN() on the one
330 * hand or initialized with init_srcu_struct_fast_updown() on the other.
331 * Such an srcu_struct structure cannot be passed to any non-fast-updown
332 * variant of srcu_read_{,un}lock() or srcu_{down,up}_read(). In kernels
333 * built with CONFIG_PROVE_RCU=y, __srcu_check_read_flavor() will complain
334 * bitterly if you ignore this * restriction.
335 *
336 * Grace-period auto-expediting is disabled for SRCU-fast-updown
337 * srcu_struct structures because SRCU-fast-updown expedited grace periods
338 * invoke synchronize_rcu_expedited(), IPIs and all. If you need expedited
339 * SRCU-fast-updown grace periods, use synchronize_srcu_expedited().
340 *
341 * The srcu_read_lock_fast_updown() function can be invoked only from
342 * those contexts where RCU is watching, that is, from contexts where
343 * it would be legal to invoke rcu_read_lock(). Otherwise, lockdep will
344 * complain.
345 */
346static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp)
347__acquires(ssp)
348{
349 struct srcu_ctr __percpu *retval;
350
351 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown().");
352 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
353 retval = __srcu_read_lock_fast_updown(ssp);
354 rcu_try_lock_acquire(&ssp->dep_map);
355 return retval;
356}
357
358/*
359 * Used by tracing, cannot be traced and cannot call lockdep.
360 * See srcu_read_lock_fast() for more information.
361 */
362static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp)
363 __acquires(ssp)
364{
365 struct srcu_ctr __percpu *retval;
366
367 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
368 retval = __srcu_read_lock_fast(ssp);
369 return retval;
370}
371
372/**
373 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
374 * @ssp: srcu_struct in which to register the new reader.
375 *
376 * Enter a semaphore-like SRCU read-side critical section, but for
377 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and
378 * srcu_down_read() for more information.
379 *
380 * The same srcu_struct may be used concurrently by srcu_down_read_fast()
381 * and srcu_read_lock_fast(). However, the same definition/initialization
382 * requirements called out for srcu_read_lock_safe() apply.
383 */
384static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)
385{
386 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
387 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");
388 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
389 return __srcu_read_lock_fast_updown(ssp);
390}
391
392/**
393 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
394 * @ssp: srcu_struct in which to register the new reader.
395 *
396 * Enter an SRCU read-side critical section, but in an NMI-safe manner.
397 * See srcu_read_lock() for more information.
398 *
399 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
400 * then none of the other flavors may be used, whether before, during,
401 * or after.
402 */
403static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
404{
405 int retval;
406
407 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
408 retval = __srcu_read_lock_nmisafe(ssp);
409 rcu_try_lock_acquire(&ssp->dep_map);
410 return retval;
411}
412
413/* Used by tracing, cannot be traced and cannot invoke lockdep. */
414static inline notrace int
415srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
416{
417 int retval;
418
419 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
420 retval = __srcu_read_lock(ssp);
421 return retval;
422}
423
424/**
425 * srcu_down_read - register a new reader for an SRCU-protected structure.
426 * @ssp: srcu_struct in which to register the new reader.
427 *
428 * Enter a semaphore-like SRCU read-side critical section. Note that
429 * SRCU read-side critical sections may be nested. However, it is
430 * illegal to call anything that waits on an SRCU grace period for the
431 * same srcu_struct, whether directly or indirectly. Please note that
432 * one way to indirectly wait on an SRCU grace period is to acquire
433 * a mutex that is held elsewhere while calling synchronize_srcu() or
434 * synchronize_srcu_expedited(). But if you want lockdep to help you
435 * keep this stuff straight, you should instead use srcu_read_lock().
436 *
437 * The semaphore-like nature of srcu_down_read() means that the matching
438 * srcu_up_read() can be invoked from some other context, for example,
439 * from some other task or from an irq handler. However, neither
440 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
441 *
442 * Calls to srcu_down_read() may be nested, similar to the manner in
443 * which calls to down_read() may be nested. The same srcu_struct may be
444 * used concurrently by srcu_down_read() and srcu_read_lock().
445 */
446static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
447{
448 WARN_ON_ONCE(in_nmi());
449 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
450 return __srcu_read_lock(ssp);
451}
452
453/**
454 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
455 * @ssp: srcu_struct in which to unregister the old reader.
456 * @idx: return value from corresponding srcu_read_lock().
457 *
458 * Exit an SRCU read-side critical section.
459 */
460static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
461 __releases(ssp)
462{
463 WARN_ON_ONCE(idx & ~0x1);
464 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
465 srcu_lock_release(&ssp->dep_map);
466 __srcu_read_unlock(ssp, idx);
467}
468
469/**
470 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
471 * @ssp: srcu_struct in which to unregister the old reader.
472 * @scp: return value from corresponding srcu_read_lock_fast().
473 *
474 * Exit a light-weight SRCU read-side critical section.
475 */
476static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
477 __releases(ssp)
478{
479 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
480 srcu_lock_release(&ssp->dep_map);
481 __srcu_read_unlock_fast(ssp, scp);
482 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");
483}
484
485/**
486 * srcu_read_unlock_fast_updown - unregister a old reader from an SRCU-fast-updown structure.
487 * @ssp: srcu_struct in which to unregister the old reader.
488 * @scp: return value from corresponding srcu_read_lock_fast_updown().
489 *
490 * Exit an SRCU-fast-updown read-side critical section.
491 */
492static inline void
493srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases(ssp)
494{
495 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
496 srcu_lock_release(&ssp->dep_map);
497 __srcu_read_unlock_fast_updown(ssp, scp);
498 RCU_LOCKDEP_WARN(!rcu_is_watching(),
499 "RCU must be watching srcu_read_unlock_fast_updown().");
500}
501
502/*
503 * Used by tracing, cannot be traced and cannot call lockdep.
504 * See srcu_read_unlock_fast() for more information.
505 */
506static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp,
507 struct srcu_ctr __percpu *scp) __releases(ssp)
508{
509 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
510 __srcu_read_unlock_fast(ssp, scp);
511}
512
513/**
514 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
515 * @ssp: srcu_struct in which to unregister the old reader.
516 * @scp: return value from corresponding srcu_read_lock_fast().
517 *
518 * Exit an SRCU read-side critical section, but not necessarily from
519 * the same context as the maching srcu_down_read_fast().
520 */
521static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
522 __releases(ssp)
523{
524 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
525 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
526 __srcu_read_unlock_fast_updown(ssp, scp);
527 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown().");
528}
529
530/**
531 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
532 * @ssp: srcu_struct in which to unregister the old reader.
533 * @idx: return value from corresponding srcu_read_lock_nmisafe().
534 *
535 * Exit an SRCU read-side critical section, but in an NMI-safe manner.
536 */
537static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
538 __releases(ssp)
539{
540 WARN_ON_ONCE(idx & ~0x1);
541 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
542 rcu_lock_release(&ssp->dep_map);
543 __srcu_read_unlock_nmisafe(ssp, idx);
544}
545
546/* Used by tracing, cannot be traced and cannot call lockdep. */
547static inline notrace void
548srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
549{
550 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
551 __srcu_read_unlock(ssp, idx);
552}
553
554/**
555 * srcu_up_read - unregister a old reader from an SRCU-protected structure.
556 * @ssp: srcu_struct in which to unregister the old reader.
557 * @idx: return value from corresponding srcu_read_lock().
558 *
559 * Exit an SRCU read-side critical section, but not necessarily from
560 * the same context as the maching srcu_down_read().
561 */
562static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
563 __releases(ssp)
564{
565 WARN_ON_ONCE(idx & ~0x1);
566 WARN_ON_ONCE(in_nmi());
567 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
568 __srcu_read_unlock(ssp, idx);
569}
570
571/**
572 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
573 *
574 * Converts the preceding srcu_read_unlock into a two-way memory barrier.
575 *
576 * Call this after srcu_read_unlock, to guarantee that all memory operations
577 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
578 * the preceding srcu_read_unlock.
579 */
580static inline void smp_mb__after_srcu_read_unlock(void)
581{
582 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
583}
584
585/**
586 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
587 *
588 * Converts the preceding srcu_read_lock into a two-way memory barrier.
589 *
590 * Call this after srcu_read_lock, to guarantee that all memory operations
591 * that occur after smp_mb__after_srcu_read_lock will appear to happen after
592 * the preceding srcu_read_lock.
593 */
594static inline void smp_mb__after_srcu_read_lock(void)
595{
596 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */
597}
598
599DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
600 _T->idx = srcu_read_lock(_T->lock),
601 srcu_read_unlock(_T->lock, _T->idx),
602 int idx)
603
604DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct,
605 _T->scp = srcu_read_lock_fast(_T->lock),
606 srcu_read_unlock_fast(_T->lock, _T->scp),
607 struct srcu_ctr __percpu *scp)
608
609DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,
610 _T->scp = srcu_read_lock_fast_notrace(_T->lock),
611 srcu_read_unlock_fast_notrace(_T->lock, _T->scp),
612 struct srcu_ctr __percpu *scp)
613
614#endif