tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

srcu: Create a DEFINE_SRCU_FAST()

This commit creates DEFINE_SRCU_FAST() and DEFINE_STATIC_SRCU_FAST()
macros that are similar to DEFINE_SRCU() and DEFINE_STATIC_SRCU(),
but which create srcu_struct structures that are usable only by readers
initiated by srcu_read_lock_fast() and friends.

This commit does make DEFINE_SRCU_FAST() available to modules, in which
case the per-CPU srcu_data structures are not created at compile time, but
rather at module-load time. This means that the >srcu_reader_flavor field
of the srcu_data structure is not available. Therefore,
this commit instead creates an ->srcu_reader_flavor field in the
srcu_struct structure, adds arguments to the DEFINE_SRCU()-related
macros to initialize this new field, and extends the checks in the
__srcu_check_read_flavor() function to include this new field.

This commit also allows dynamically allocated srcu_struct structure
to be marked for SRCU-fast readers. It does so by defining a new
init_srcu_struct_fast() function that marks the specified srcu_struct
structure for use by srcu_read_lock_fast() and friends.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

authored by

Paul E. McKenney and committed by
Frederic Weisbecker ee908484 950063c6

+78 -19
+1 -1
include/linux/notifier.h
··· 109 109 .mutex = __MUTEX_INITIALIZER(name.mutex), \ 110 110 .head = NULL, \ 111 111 .srcuu = __SRCU_USAGE_INIT(name.srcuu), \ 112 - .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ 112 + .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu, 0), \ 113 113 } 114 114 115 115 #define ATOMIC_NOTIFIER_HEAD(name) \
+14 -2
include/linux/srcu.h
··· 25 25 26 26 #ifdef CONFIG_DEBUG_LOCK_ALLOC 27 27 28 - int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 29 - struct lock_class_key *key); 28 + int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 29 + #ifndef CONFIG_TINY_SRCU 30 + int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 31 + #endif // #ifndef CONFIG_TINY_SRCU 30 32 31 33 #define init_srcu_struct(ssp) \ 32 34 ({ \ ··· 37 35 __init_srcu_struct((ssp), #ssp, &__srcu_key); \ 38 36 }) 39 37 38 + #define init_srcu_struct_fast(ssp) \ 39 + ({ \ 40 + static struct lock_class_key __srcu_key; \ 41 + \ 42 + __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \ 43 + }) 44 + 40 45 #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, 41 46 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 42 47 43 48 int init_srcu_struct(struct srcu_struct *ssp); 49 + #ifndef CONFIG_TINY_SRCU 50 + int init_srcu_struct_fast(struct srcu_struct *ssp); 51 + #endif // #ifndef CONFIG_TINY_SRCU 44 52 45 53 #define __SRCU_DEP_MAP_INIT(srcu_name) 46 54 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
+10 -3
include/linux/srcutiny.h
··· 31 31 32 32 void srcu_drive_gp(struct work_struct *wp); 33 33 34 - #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored) \ 34 + #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ 35 35 { \ 36 36 .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ 37 37 .srcu_cb_tail = &name.srcu_cb_head, \ ··· 44 44 * Tree SRCU, which needs some per-CPU data. 45 45 */ 46 46 #define DEFINE_SRCU(name) \ 47 - struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) 47 + struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) 48 48 #define DEFINE_STATIC_SRCU(name) \ 49 - static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) 49 + static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) 50 + #define DEFINE_SRCU_FAST(name) DEFINE_SRCU(name) 51 + #define DEFINE_STATIC_SRCU_FAST(name) \ 52 + static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) 50 53 51 54 // Dummy structure for srcu_notifier_head. 52 55 struct srcu_usage { }; 53 56 #define __SRCU_USAGE_INIT(name) { } 57 + #define __init_srcu_struct_fast __init_srcu_struct 58 + #ifndef CONFIG_DEBUG_LOCK_ALLOC 59 + #define init_srcu_struct_fast init_srcu_struct 60 + #endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC 54 61 55 62 void synchronize_srcu(struct srcu_struct *ssp); 56 63
+19 -11
include/linux/srcutree.h
··· 104 104 struct srcu_struct { 105 105 struct srcu_ctr __percpu *srcu_ctrp; 106 106 struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */ 107 + u8 srcu_reader_flavor; 107 108 struct lockdep_map dep_map; 108 109 struct srcu_usage *srcu_sup; /* Update-side data. */ 109 110 }; ··· 163 162 .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ 164 163 } 165 164 166 - #define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ 165 + #define __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \ 167 166 .srcu_sup = &usage_name, \ 167 + .srcu_reader_flavor = fast, \ 168 168 __SRCU_DEP_MAP_INIT(name) 169 169 170 - #define __SRCU_STRUCT_INIT_MODULE(name, usage_name) \ 170 + #define __SRCU_STRUCT_INIT_MODULE(name, usage_name, fast) \ 171 171 { \ 172 - __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ 172 + __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \ 173 173 } 174 174 175 - #define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name) \ 175 + #define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name, fast) \ 176 176 { \ 177 177 .sda = &pcpu_name, \ 178 178 .srcu_ctrp = &pcpu_name.srcu_ctrs[0], \ 179 - __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ 179 + __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \ 180 180 } 181 181 182 182 /* ··· 198 196 * init_srcu_struct(&my_srcu); 199 197 * 200 198 * See include/linux/percpu-defs.h for the rules on per-CPU variables. 199 + * 200 + * DEFINE_SRCU_FAST() creates an srcu_struct and associated structures 201 + * whose readers must be of the SRCU-fast variety. 201 202 */ 202 203 #ifdef MODULE 203 - # define __DEFINE_SRCU(name, is_static) \ 204 + # define __DEFINE_SRCU(name, fast, is_static) \ 204 205 static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage); \ 205 - is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \ 206 + is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage, \ 207 + fast); \ 206 208 extern struct srcu_struct * const __srcu_struct_##name; \ 207 209 struct srcu_struct * const __srcu_struct_##name \ 208 210 __section("___srcu_struct_ptrs") = &name 209 211 #else 210 - # define __DEFINE_SRCU(name, is_static) \ 212 + # define __DEFINE_SRCU(name, fast, is_static) \ 211 213 static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \ 212 214 static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage); \ 213 215 is_static struct srcu_struct name = \ 214 - __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data) 216 + __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast) 215 217 #endif 216 - #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) 217 - #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) 218 + #define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* not static */) 219 + #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, 0, static) 220 + #define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */) 221 + #define DEFINE_STATIC_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static) 218 222 219 223 int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); 220 224 void synchronize_srcu_expedited(struct srcu_struct *ssp);
+34 -2
kernel/rcu/srcutree.c
··· 286 286 287 287 #ifdef CONFIG_DEBUG_LOCK_ALLOC 288 288 289 - int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 290 - struct lock_class_key *key) 289 + static int 290 + __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 291 291 { 292 292 /* Don't re-initialize a lock while it is held. */ 293 293 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); 294 294 lockdep_init_map(&ssp->dep_map, name, key, 0); 295 295 return init_srcu_struct_fields(ssp, false); 296 296 } 297 + 298 + int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 299 + { 300 + ssp->srcu_reader_flavor = 0; 301 + return __init_srcu_struct_common(ssp, name, key); 302 + } 297 303 EXPORT_SYMBOL_GPL(__init_srcu_struct); 304 + 305 + int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 306 + { 307 + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; 308 + return __init_srcu_struct_common(ssp, name, key); 309 + } 310 + EXPORT_SYMBOL_GPL(__init_srcu_struct_fast); 298 311 299 312 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 300 313 ··· 321 308 */ 322 309 int init_srcu_struct(struct srcu_struct *ssp) 323 310 { 311 + ssp->srcu_reader_flavor = 0; 324 312 return init_srcu_struct_fields(ssp, false); 325 313 } 326 314 EXPORT_SYMBOL_GPL(init_srcu_struct); 315 + 316 + /** 317 + * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure 318 + * @ssp: structure to initialize. 319 + * 320 + * Must invoke this on a given srcu_struct before passing that srcu_struct 321 + * to any other function. Each srcu_struct represents a separate domain 322 + * of SRCU protection. 323 + */ 324 + int init_srcu_struct_fast(struct srcu_struct *ssp) 325 + { 326 + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; 327 + return init_srcu_struct_fields(ssp, false); 328 + } 329 + EXPORT_SYMBOL_GPL(init_srcu_struct_fast); 327 330 328 331 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 329 332 ··· 763 734 764 735 sdp = raw_cpu_ptr(ssp->sda); 765 736 old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor); 737 + WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor); 738 + WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor && 739 + old_read_flavor != ssp->srcu_reader_flavor); 766 740 if (!old_read_flavor) { 767 741 old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor); 768 742 if (!old_read_flavor)