rcu: Invert passed_quiesce and rename to cpu_no_qs

Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git

kernel os linux

This commit inverts the sense of the rcu_data structure's ->passed_quiesce
field and renames it to ->cpu_no_qs. This will allow a later commit to
use an "aggregate OR" operation to test expedited as well as normal grace
periods without added overhead.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Paul E. McKenney 10 years ago 0d43eb34 97c668b8

+33 -33

5 changed files

expand all

Documentation

RCU

trace.txt

kernel

rcu

tree.c

tree.h

tree_plugin.h

tree_trace.c

+16 -16

Documentation/RCU/trace.txt

··· 56 56 57 57 The output of "cat rcu/rcu_preempt/rcudata" looks as follows: 58 58 59 - 0!c=30455 g=30456 pq=1/0 qp=1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716 60 - 1!c=30719 g=30720 pq=1/0 qp=0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982 61 - 2!c=30150 g=30151 pq=1/1 qp=1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458 62 - 3 c=31249 g=31250 pq=1/1 qp=0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622 63 - 4!c=29502 g=29503 pq=1/0 qp=1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521 64 - 5 c=31201 g=31202 pq=1/0 qp=1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698 65 - 6!c=30253 g=30254 pq=1/0 qp=1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353 66 - 7 c=31178 g=31178 pq=1/0 qp=0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969 59 + 0!c=30455 g=30456 cnq=1/0:1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716 60 + 1!c=30719 g=30720 cnq=1/0:0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982 61 + 2!c=30150 g=30151 cnq=1/1:1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458 62 + 3 c=31249 g=31250 cnq=1/1:0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622 63 + 4!c=29502 g=29503 cnq=1/0:1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521 64 + 5 c=31201 g=31202 cnq=1/0:1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698 65 + 6!c=30253 g=30254 cnq=1/0:1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353 66 + 7 c=31178 g=31178 cnq=1/0:0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969 67 67 68 68 This file has one line per CPU, or eight for this 8-CPU system. 69 69 The fields are as follows: ··· 188 188 Kernels compiled with CONFIG_RCU_BOOST=y display the following from 189 189 /debug/rcu/rcu_preempt/rcudata: 190 190 191 - 0!c=12865 g=12866 pq=1/0 qp=1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871 192 - 1 c=14407 g=14408 pq=1/0 qp=0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485 193 - 2 c=14407 g=14408 pq=1/0 qp=0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490 194 - 3 c=14407 g=14408 pq=1/0 qp=0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290 195 - 4 c=14405 g=14406 pq=1/0 qp=1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114 196 - 5!c=14168 g=14169 pq=1/0 qp=0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722 197 - 6 c=14404 g=14405 pq=1/0 qp=0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811 198 - 7 c=14407 g=14408 pq=1/0 qp=1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042 191 + 0!c=12865 g=12866 cnq=1/0:1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871 192 + 1 c=14407 g=14408 cnq=1/0:0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485 193 + 2 c=14407 g=14408 cnq=1/0:0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490 194 + 3 c=14407 g=14408 cnq=1/0:0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290 195 + 4 c=14405 g=14406 cnq=1/0:1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114 196 + 5!c=14168 g=14169 cnq=1/0:0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722 197 + 6 c=14404 g=14405 cnq=1/0:0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811 198 + 7 c=14407 g=14408 cnq=1/0:1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042 199 199 200 200 This is similar to the output discussed above, but contains the following 201 201 additional fields:

+11 -11

kernel/rcu/tree.c

··· 245 245 */ 246 246 void rcu_sched_qs(void) 247 247 { 248 - if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) { 248 + if (__this_cpu_read(rcu_sched_data.cpu_no_qs)) { 249 249 trace_rcu_grace_period(TPS("rcu_sched"), 250 250 __this_cpu_read(rcu_sched_data.gpnum), 251 251 TPS("cpuqs")); 252 - __this_cpu_write(rcu_sched_data.passed_quiesce, 1); 252 + __this_cpu_write(rcu_sched_data.cpu_no_qs, false); 253 253 } 254 254 } 255 255 256 256 void rcu_bh_qs(void) 257 257 { 258 - if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { 258 + if (__this_cpu_read(rcu_bh_data.cpu_no_qs)) { 259 259 trace_rcu_grace_period(TPS("rcu_bh"), 260 260 __this_cpu_read(rcu_bh_data.gpnum), 261 261 TPS("cpuqs")); 262 - __this_cpu_write(rcu_bh_data.passed_quiesce, 1); 262 + __this_cpu_write(rcu_bh_data.cpu_no_qs, false); 263 263 } 264 264 } 265 265 ··· 1744 1744 */ 1745 1745 rdp->gpnum = rnp->gpnum; 1746 1746 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); 1747 - rdp->passed_quiesce = 0; 1747 + rdp->cpu_no_qs = true; 1748 1748 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); 1749 1749 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); 1750 1750 zero_cpu_stall_ticks(rdp); ··· 2337 2337 rnp = rdp->mynode; 2338 2338 raw_spin_lock_irqsave(&rnp->lock, flags); 2339 2339 smp_mb__after_unlock_lock(); 2340 - if ((rdp->passed_quiesce == 0 && 2340 + if ((rdp->cpu_no_qs && 2341 2341 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || 2342 2342 rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || 2343 2343 rdp->gpwrap) { ··· 2348 2348 * We will instead need a new quiescent state that lies 2349 2349 * within the current grace period. 2350 2350 */ 2351 - rdp->passed_quiesce = 0; /* need qs for new gp. */ 2351 + rdp->cpu_no_qs = true; /* need qs for new gp. */ 2352 2352 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); 2353 2353 raw_spin_unlock_irqrestore(&rnp->lock, flags); 2354 2354 return; ··· 2395 2395 * Was there a quiescent state since the beginning of the grace 2396 2396 * period? If no, then exit and wait for the next call. 2397 2397 */ 2398 - if (!rdp->passed_quiesce && 2398 + if (rdp->cpu_no_qs && 2399 2399 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) 2400 2400 return; 2401 2401 ··· 3828 3828 3829 3829 /* Is the RCU core waiting for a quiescent state from this CPU? */ 3830 3830 if (rcu_scheduler_fully_active && 3831 - rdp->core_needs_qs && !rdp->passed_quiesce && 3831 + rdp->core_needs_qs && rdp->cpu_no_qs && 3832 3832 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { 3833 3833 rdp->n_rp_core_needs_qs++; 3834 3834 } else if (rdp->core_needs_qs && 3835 - (rdp->passed_quiesce || 3835 + (!rdp->cpu_no_qs || 3836 3836 rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) { 3837 3837 rdp->n_rp_report_qs++; 3838 3838 return 1; ··· 4155 4155 rdp->beenonline = true; /* We have now been online. */ 4156 4156 rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ 4157 4157 rdp->completed = rnp->completed; 4158 - rdp->passed_quiesce = false; 4158 + rdp->cpu_no_qs = true; 4159 4159 rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu); 4160 4160 rdp->core_needs_qs = false; 4161 4161 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

+1 -1

kernel/rcu/tree.h

··· 302 302 /* is aware of having started. */ 303 303 unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */ 304 304 /* for rcu_all_qs() invocations. */ 305 - bool passed_quiesce; /* User-mode/idle loop etc. */ 305 + bool cpu_no_qs; /* No QS yet for this CPU. */ 306 306 bool core_needs_qs; /* Core waits for quiesc state. */ 307 307 bool beenonline; /* CPU online at least once. */ 308 308 bool gpwrap; /* Possible gpnum/completed wrap. */

+3 -3

kernel/rcu/tree_plugin.h

··· 265 265 */ 266 266 static void rcu_preempt_qs(void) 267 267 { 268 - if (!__this_cpu_read(rcu_data_p->passed_quiesce)) { 268 + if (__this_cpu_read(rcu_data_p->cpu_no_qs)) { 269 269 trace_rcu_grace_period(TPS("rcu_preempt"), 270 270 __this_cpu_read(rcu_data_p->gpnum), 271 271 TPS("cpuqs")); 272 - __this_cpu_write(rcu_data_p->passed_quiesce, 1); 272 + __this_cpu_write(rcu_data_p->cpu_no_qs, false); 273 273 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */ 274 274 current->rcu_read_unlock_special.b.need_qs = false; 275 275 } ··· 620 620 } 621 621 if (t->rcu_read_lock_nesting > 0 && 622 622 __this_cpu_read(rcu_data_p->core_needs_qs) && 623 - !__this_cpu_read(rcu_data_p->passed_quiesce)) 623 + __this_cpu_read(rcu_data_p->cpu_no_qs)) 624 624 t->rcu_read_unlock_special.b.need_qs = true; 625 625 } 626 626

+2 -2

kernel/rcu/tree_trace.c

··· 117 117 118 118 if (!rdp->beenonline) 119 119 return; 120 - seq_printf(m, "%3d%cc=%ld g=%ld pq=%d/%d qp=%d", 120 + seq_printf(m, "%3d%cc=%ld g=%ld cnq=%d/%d:%d", 121 121 rdp->cpu, 122 122 cpu_is_offline(rdp->cpu) ? '!' : ' ', 123 123 ulong2long(rdp->completed), ulong2long(rdp->gpnum), 124 - rdp->passed_quiesce, 124 + rdp->cpu_no_qs, 125 125 rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu), 126 126 rdp->core_needs_qs); 127 127 seq_printf(m, " dt=%d/%llx/%d df=%lu",