From 29871903f3a38e75e896f9a644713859490cb85f Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 1 Sep 2026 20:43:47 +0800 Subject: [PATCH 01/18] sched_ext: Don't deliver duplicate ops.cgroup_set_idle() for same value ops.cgroup_set_idle() is documented to be invoked when a cgroup transitions between idle and non-idle states, and scx_group_set_weight() already skips value-preserving writes. scx_group_set_idle() delivers every write unconditionally, so rewriting an already-correct cpu.idle value feeds the BPF scheduler a transition callback each time, which toggle- or accounting-based schedulers miscount. Mirror the weight guard and only deliver on an actual change. Verified with a probe scheduler printing each callback: rewriting cpu.idle=1 twice on an already-idle cgroup delivered two callbacks before and none after. Fixes: 347ed2d566da ("sched/ext: Implement cgroup_set_idle() callback") Link: https://lore.kernel.org/r/b53c61a1-4d7d-4232-941f-d48b0563d4ed@linux.dev Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 901fb0f8b976..5f242ab69cf4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4933,7 +4933,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && + tg->scx.idle != idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ -- knotmirror 0.0.1 From 0a85182723b65ad8bee8131bc38fcf0347d6679b Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 14:06:26 +0800 Subject: [PATCH 02/18] sched_ext: Fix NULL sched deref in kfunc sub-sched error paths When the root scheduler has sub-scheds attached, the COMPAT kfunc wrappers scx_bpf_select_cpu_and() and scx_bpf_dsq_insert_vtime() refuse the call and report to @p's scheduler: scx_error(scx_task_sched(p), "... must be used"); The wrappers are reachable with tasks that have no scheduler. scx_bpf_select_cpu_and() is in the select_cpu kfunc group, which scx_kfunc_context_filter() opens to BPF_PROG_TYPE_SYSCALL programs; scx_bpf_dsq_insert_vtime() is in the enqueue_dispatch group, which ops.enqueue() and ops.dispatch() may call with any KF_RCU task -- the group has no kf_tasks validation, and scx_dsq_insert_preamble() checks task ownership with scx_task_on_sched() precisely because @p may be an arbitrary task. scx_task_sched(p) is p->scx.sched, which is NULL for tasks past sched_ext_dead() -- which clears it via scx_disable_and_exit_task() on exit -- and for idle tasks, which the enable paths skip as they are never scheduled through SCX. It is also an rcu_dereference_protected() that expects @p's pi_lock or rq lock, which neither wrapper holds. Passing NULL to scx_error() reaches scx_vexit(), which dereferences sch->exit_info, oopsing the kernel. One concrete trigger exercised while developing the fix: a BPF_PROG_TYPE_SYSCALL program calling the select_cpu_and wrapper on an exited-but-not-reaped task while a sub-scheduler was attached (its pid stays findable while the zombie is unreaped; faulting instruction is the scx_vexit() prologue "mov r15,[rdi+0x398]" with RDI=NULL and 0x398 the offset of sch->exit_info): sched_ext: BPF scheduler "kfunc_subsched_null" enabled sched_ext: BPF sub-scheduler "kfunc_subsched_null" enabled sched_ext: Unassociated program run_select_cpu_ (id 76) BUG: kernel NULL pointer dereference, address: 0000000000000398 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page Oops: Oops: 0000 [#1] SMP NOPTI CPU: 7 UID: 0 PID: 8201 Comm: kfunc_test_runn Tainted: G W RIP: 0010:scx_vexit+0x25/0xa0 Code: ... <4c> 8b bf 98 03 00 00 ... CR2: 0000000000000398 Call Trace: __scx_exit+0x4f/0x70 scx_bpf_select_cpu_and+0xab/0xb0 bpf_prog_430ed61a7b66e03a_run_select_cpu_and+0x9c/0xe7 ? __x64_sys_bpf+0x2c/0x40 bpf_prog_test_run_syscall+0x130/0x2f0 __sys_bpf+0x930/0x10d0 ? __x64_sys_bpf+0x2c/0x40 __x64_sys_bpf+0x2c/0x40 do_syscall_64+0xbc/0x460 entry_SYSCALL_64_after_hwframe+0x76/0x7e Read @p's scheduler under RCU instead, which the wrappers can do from their guard(rcu)(): fault it when it can be determined, and when it can't be determined -- @p is a task past sched_ext_dead() or an idle task -- there is nothing obviously wrong to report, so just refuse the call as before without faulting any scheduler. These COMPAT wrappers are scheduled for eventual removal once the deprecation grace period elapses, but until then -- and regardless of their removal timeline -- they must not oops the kernel on a task they are handed. Cc: stable@vger.kernel.org # v7.1+ Fixes: a5fa0708cbfd ("sched_ext: Enforce scheduling authority in dispatch and select_cpu operations") Suggested-by: Andrea Righi Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++-- kernel/sched/ext/idle.c | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5f242ab69cf4..7e414a7c53fc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8949,10 +8949,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used"); return; } #endif diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d2973fb3af6d..aa9fb6de0ad6 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_select_cpu_and() must be used"); return -EINVAL; } #endif -- knotmirror 0.0.1 From 3265ef0b670180b0b946d73ee9825d6d91e98a08 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:55 -1000 Subject: [PATCH 03/18] sched_ext: Rename sch to root_sch in dispatch_one() dispatch_one() uses the root scheduler for everything it does, including the two decisions to keep running @prev, which are wrong when @prev belongs to a sub-scheduler. The function has to deal with @prev's scheduler too. Rename the root's local from sch to root_sch for clarity and to make room for it. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7e414a7c53fc..120540cdda74 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq) static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) { - struct scx_sched *sch = scx_root_protected_live(); + struct scx_sched *root_sch = scx_root_protected_live(); enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); @@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev scx_process_sync_ecaps(rq, prev); - if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && + if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { /* * If the previous sched_class for the current CPU was not SCX, @@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ - if (sch->ops.cpu_acquire) - SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); + if (root_sch->ops.cpu_acquire) + SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL); rq->scx.cpu_released = false; } @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(sch, cpu)) { + !scx_bypassing(root_sch, cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2967,7 +2967,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; } - verdict = scx_dispatch_sched(sch, rq, prev, false); + verdict = scx_dispatch_sched(root_sch, rq, prev, false); if (verdict != SCX_DSP_NONE) goto has_tasks; @@ -2976,9 +2976,9 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); verdict = SCX_DSP_PREV; goto has_tasks; } -- knotmirror 0.0.1 From 90f19b2816f5d243a8daf36ca83d9d9d04f00e4c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:56 -1000 Subject: [PATCH 04/18] sched_ext: Use @prev's scheduler for the keep decisions in dispatch_one() dispatch_one() tests ops flags and bypass state against the root scheduler in both places where it decides to keep running @prev: the early keep of a @prev with slice left tests the root's bypass state, and the keep-last at the end tests the root's SCX_OPS_ENQ_LAST and bypass state. Both are properties of the scheduler @prev belongs to, and put_prev_task_scx(), which acts on the outcome, reads them from that scheduler. When @prev belongs to a sub-scheduler the two sides disagree. The keep-last case is visible. The root set SCX_OPS_ENQ_LAST, so a lone @prev of a sub-scheduler is not kept and is enqueued with SCX_ENQ_LAST to a sub-scheduler that never opted in. This trips the WARN_ON_ONCE in put_prev_task_scx() for the missing flag, and the sub-scheduler queues the task like any other and triggers no follow-up scheduling event, which can lead to stalls. Test SCX_OPS_ENQ_LAST and bypass state on @prev's sched in both places and charge SCX_EV_DISPATCH_KEEP_LAST to it. Read the sched at each decision, as the dispatch in between can drop the rq lock. Fixes: 88234b075c3f ("sched_ext: Introduce scx_task_sched[_rcu]()") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 120540cdda74..adf5993fa597 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(root_sch, cpu)) { + !scx_bypassing(scx_task_sched(prev), cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2972,15 +2972,20 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; /* - * Didn't find another task to run. Keep running @prev unless - * %SCX_OPS_ENQ_LAST is in effect. + * Didn't find another task to run. Keep running @prev unless its own + * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see + * put_prev_task_scx(). Read the scheduler here as the dispatch above + * may have dropped the rq lock while @prev changed class or scheduler. */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && - scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); - verdict = SCX_DSP_PREV; - goto has_tasks; + if (prev->scx.flags & SCX_TASK_QUEUED) { + struct scx_sched *prev_sch = scx_task_sched(prev); + + if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) || + scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { + __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; + goto has_tasks; + } } rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; -- knotmirror 0.0.1 From a0d356696f87700c8c2934e3881277b0d37f0b71 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:57 -1000 Subject: [PATCH 05/18] sched_ext: scx_qmap: Do not add IMMED to rescue inserts qmap's stranded fallback forces a task that can run on none of its self cids onto its first allowed cid with SCX_ENQ_RESCUE, and adds SCX_ENQ_IMMED when that cid is a time-share it holds. On such a cid the insert stops being a rescue request: 1. A task is enqueued while none of its allowed cids is in self_cids. At attach self_cids is still empty. 2. qmap inserts it into cid 0's local DSQ with SCX_ENQ_RESCUE | SCX_ENQ_IMMED. 3. The kernel finds ENQ_IMMED held on cid 0, admits the insert and skips the rescue diversion. 4. cid 0's cpu is busy, so the IMMED task is bounced back to qmap with SCX_ENQ_REENQ. 5. qmap's enqueue sees the same inputs and repeats step 2. Nothing runs in between. 6. The reenqueue limit ejects qmap with SCX_EXIT_ERROR_REENQ. The caps granted during the parent's ops.sub_attach() are delivered after the sub already holds its tasks, while the per-cid effective caps that mark the time-shares are delivered from the first dispatch after bypass lifts, so every attach that receives a time-share on a task's first allowed cid starts the loop. Drop IMMED from the rescue inserts so that step 3 diverts to the rescue path. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 9f6e61d7ca07..e4e51303bd29 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -358,8 +358,8 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p, } /* - * A received time-shared cid is held ENQ_IMMED-only, so inserts must set - * SCX_ENQ_IMMED. + * A received time-shared cid is held ENQ_IMMED-only, so inserts meant to run + * there must set SCX_ENQ_IMMED. */ static u64 needs_immed(s32 cid) { @@ -444,9 +444,11 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) * didn't grant them or we delegated them to children - would starve in * SHARED/FIFO since we only pull from those on self cids. * - * Force it onto its first allowed cid's local DSQ. If we hold that cid - * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel - * diverts the task to its rescue path. + * Force it onto its first allowed cid's local DSQ with SCX_ENQ_RESCUE. + * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the + * task to its rescue path. IMMED would turn the insert into a legal + * placement on a time-shared cid and the kernel would bounce it back + * here instead of rescuing it. */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); @@ -455,7 +457,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) taskc->force_local = false; __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, - enq_flags | needs_immed(c) | SCX_ENQ_RESCUE); + enq_flags | SCX_ENQ_RESCUE); return; } } @@ -618,7 +620,7 @@ static bool scan_shared_dsq(bool from_timer) if (c >= 0 && c < scx_bpf_nr_cids()) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - needs_immed(c) | SCX_ENQ_RESCUE); + SCX_ENQ_RESCUE); } continue; } @@ -659,7 +661,7 @@ static bool scan_shared_dsq(bool from_timer) if (c >= 0 && c < nr_cids) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - needs_immed(c) | SCX_ENQ_RESCUE); + SCX_ENQ_RESCUE); } continue; } -- knotmirror 0.0.1 From 63b4ff622244483e7c530e97d787a3d6c2c38a33 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:58 -1000 Subject: [PATCH 06/18] sched_ext: scx_qmap: Place only on cids whose caps are in effect qmap decides placements from self_cids, which redistribute() derives from the caps view at ops.sub_caps_updated() time. That view runs ahead of the cpus: a granted cid can be in self_cids before its cpu has reported the caps in effect through ops.sub_ecaps_updated(). ops.update_idle() only comes once BASE is in effect, so the idle-gated placements reach such a cid only through an idle bit left over from an earlier hold. The highpri scan has no gate at all: parent cpu Y, qmap cpu X grants ENQ on X to qmap sub_caps_updated() adds X to self_cids highpri scan moves a task to X with PREEMPT caps not in effect, move denied, task bounced with REENQ_CAP reject drain, enqueue the scan moves it to X again denied again dispatch syncs ecaps, sub_ecaps_updated(X) Every highpri move to X in that window is denied and bounced. The two callbacks are meant to split the roles: ops.sub_caps_updated() tracks what the node holds and drives what it delegates to its children, while ops.sub_ecaps_updated() says whether a task can run on a cpu now. qmap used the first for both. Track the caps in effect from ops.sub_ecaps_updated() as avail_cids and place only on self_cids & avail_cids, so that self_cids stays the delegation split and avail_cids gates the placement. The stranded tests keep self_cids, as they ask whether the split gives the task anywhere at all. A highpri task whose self_cids lack caps in effect waits for them instead of being moved and bounced. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 83 ++++++++++++++++++++++++++-------- tools/sched_ext/scx_qmap.h | 3 ++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index e4e51303bd29..062bb22ee65c 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -24,6 +24,9 @@ * time-share that stays self-local. * self - The excl cpus the node kept for itself, plus all of held_shared. * owner - Who holds a cid - a child slot, CID_SELF, or CID_NONE. + * avail - Cpus whose caps are in effect, per ops.sub_ecaps_updated(). + * usable - self AND avail. Placement decisions use this: self is the + * delegation split and can run ahead of what the cpus honor. * * The scheduler splits its held-excl cpus among self and the children in * proportion to each node's cpu.weight, handing each the floor of its share as @@ -208,8 +211,8 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock) } /* - * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin - * from prev_cid + 1. Atomic claim retries on race; bounded by + * Try prev_cid, then scan cpus_allowed AND idle_cids AND usable_cids + * round-robin from prev_cid + 1. Atomic claim retries on race; bounded by * IDLE_PICK_RETRIES to keep the verifier's insn budget in check. */ #define IDLE_PICK_RETRIES 16 @@ -221,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, s32 cid; u32 i; - if (cmask_test(prev_cid, &qa.self_cids.mask) && + if (cmask_test(prev_cid, &qa.usable_cids.mask) && cmask_test_and_clear(prev_cid, &qa.idle_cids.mask)) return prev_cid; @@ -229,7 +232,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, bpf_for(i, 0, IDLE_PICK_RETRIES) { cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, &qa.idle_cids.mask, - &qa.self_cids.mask, cid + 1); + &qa.usable_cids.mask, cid + 1); barrier_var(cid); if (cid >= nr_cids) return -1; @@ -542,7 +545,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, &qa.idle_cids.mask, - &qa.self_cids.mask, 0); + &qa.usable_cids.mask, 0); if (cid < scx_bpf_nr_cids()) scx_bpf_kick_cid(cid, SCX_KICK_IDLE); return; @@ -646,18 +649,23 @@ static bool scan_shared_dsq(bool from_timer) if (!(taskc = lookup_task_ctx(p))) return false; - /* only run highpri tasks on cids this node holds, not delegated ones */ + /* only run highpri tasks on cids this node can use right now */ if (cmask_test(this_cid, &taskc->cpus_allowed) && - cmask_test(this_cid, &qa.self_cids.mask)) + cmask_test(this_cid, &qa.usable_cids.mask)) cid = this_cid; else cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, - &qa.self_cids.mask, + &qa.usable_cids.mask, this_cid + 1); if (cid >= nr_cids) { - /* stranded after the cull - rescue it from here */ - s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + s32 c; + + /* self cids lack caps in effect yet, leave it queued */ + if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) + continue; + /* stranded after the cull - rescue it from here */ + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); if (c >= 0 && c < nr_cids) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, @@ -1115,7 +1123,7 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) /* * The kernel delivers update_idle() for every cid this node holds * SCX_CAP_BASE on. Track every cid's idle state regardless of - * delegation: the direct-dispatch pick masks idle_cids with self_cids + * delegation: the direct-dispatch pick masks idle_cids with usable_cids * at selection, so a cid already idle when it returns to self needs no * reseed here. */ @@ -1539,6 +1547,19 @@ static __noinline void account_alloc(void) } } +/* + * usable_cids = self_cids & avail_cids. The inputs have separate writers, + * apply_partition() and qmap_sub_ecaps_updated(), so the result is rebuilt in + * full under the partition guard, in scratch first so that readers never see + * self_cids alone. + */ +static void refresh_usable(void) +{ + cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask); + cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask); + cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask); +} + /* * apply_partition - execute the plan compute_partition() built * @@ -1561,6 +1582,7 @@ __noinline void apply_partition(void) /* no excl cpu: run own tasks on the held shares, evict children */ if (!qa.part.nr_excl) { cmask_copy(&qa.self_cids.mask, &qa.held_shared.mask); + refresh_usable(); bpf_for(i, 0, MAX_SUB_SCHEDS) if (qa.sub_sched_ctxs[i].cgroup_id) scx_bpf_sub_kill(qa.sub_sched_ctxs[i].cgroup_id, @@ -1598,6 +1620,7 @@ __noinline void apply_partition(void) else if (o == CID_SELF) cmask_set(cid, &qa.self_cids.mask); } + refresh_usable(); /* * Apply each child's exclusive cids as a delta against its previous @@ -1839,8 +1862,11 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) cmask_init(&qa.rr_cids.mask, 0, nr_cids); cmask_init(&qa.prev_rr_cids.mask, 0, nr_cids); cmask_init(&qa.self_cids.mask, 0, nr_cids); + cmask_init(&qa.avail_cids.mask, 0, nr_cids); + cmask_init(&qa.usable_cids.mask, 0, nr_cids); cmask_init(&qa.to_revoke_cids.mask, 0, nr_cids); cmask_init(&qa.to_grant_cids.mask, 0, nr_cids); + cmask_init(&qa.usable_scratch.mask, 0, nr_cids); cmask_init(&qa.held_excl.mask, 0, nr_cids); cmask_init(&qa.held_shared.mask, 0, nr_cids); @@ -1854,14 +1880,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) } /* - * The root starts holding every cid. qmap_sub_ecaps_updated() maintains - * per-cid shared state as effective caps settle, and redistribute() - * rebuilds owner and self from held caps. A non-root node starts with - * nothing. + * The root starts holding every cid and gets no ecaps notifications, so + * its avail set is fixed here. qmap_sub_ecaps_updated() maintains the + * per-cid state as effective caps settle, and redistribute() rebuilds + * owner and self from held caps. A non-root node starts with nothing. */ bpf_for(i, 0, nr_cids) { if (!sub_cgroup_id) { cmask_set(i, &qa.self_cids.mask); + cmask_set(i, &qa.avail_cids.mask); + cmask_set(i, &qa.usable_cids.mask); qa.part.cid_owner[i] = CID_SELF; } else { qa.part.cid_owner[i] = CID_NONE; @@ -2002,12 +2030,31 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) { /* * Effective caps updated. Track which cids hold shared caps so a self - * task placed there enqueues IMMED. + * task placed there enqueues IMMED, and which cids have ENQ_IMMED in + * effect at all (avail, see the header comment). */ - if (after & SCX_CAP_ENQ_IMMED) + if (after & SCX_CAP_ENQ_IMMED) { qa.cid_shared[cid] = (after & SCX_CAP_ENQ) ? 0 : 1; - else + cmask_set(cid, &qa.avail_cids.mask); + } else { qa.cid_shared[cid] = 0; + cmask_clear(cid, &qa.avail_cids.mask); + } + + /* + * When another runner holds the partition guard, set part_pending: + * redistribute() drains it before releasing and rr_advance() checks it + * after, so the deferred refresh lands by the next rr tick. A + * repartition that lost the guard to us runs here. + */ + if (part_try_start()) { + refresh_usable(); + part_end(); + if (__sync_fetch_and_or(&part_pending, 0)) + redistribute(); + } else { + __sync_fetch_and_or(&part_pending, 1); + } } SCX_OPS_CID_DEFINE(qmap_ops, diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index c78d61806b39..e95fffcf7b23 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -165,12 +165,15 @@ struct qmap_arena { /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ + struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */ + struct qmap_cmask usable_cids; /* self_cids & avail_cids, placeable right now */ struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */ struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */ /* scratch cmasks */ struct qmap_cmask to_revoke_cids; /* delta cids to revoke */ struct qmap_cmask to_grant_cids; /* delta cids to grant */ + struct qmap_cmask usable_scratch; /* refresh_usable() build area */ struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */ struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */ struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */ -- knotmirror 0.0.1 From 89ff16f0713917303210c560eec5cd0c13bd651f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 12:53:44 -1000 Subject: [PATCH 07/18] sched_ext: scx_qmap: Fix pending partition work handoff qmap can leave partition work pending with no runner. The effective-cap callback publishes its request after failing to acquire part_busy, while redistribute() checks for pending work before releasing it. Either ordering can miss a request arriving as the current runner finishes, delaying the update until the round-robin timer runs. Publish requests before trying to become the runner and release part_busy before checking for more work. Have all holders drain pending requests after releasing it, including the stats flush. Distinguish mask refreshes from repartitions so an effective-cap update only rebuilds the partition when a repartition was also requested. Fixes: e9151ed5c944 ("tools/sched_ext: scx_qmap - Expand hierarchical sub-scheduling") Reported-by: Andrea Righi Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 75 ++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 062bb22ee65c..bda56c37acb5 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1295,11 +1295,16 @@ struct { __type(value, struct round_robin_timer); } round_robin_timer SEC(".maps"); +enum part_pending_flags { + PART_REFRESH = BIT_U64(0), + PART_REDISTRIBUTE = BIT_U64(1), +}; + /* * Partition update synchronization. qa.part can be written from concurrent * contexts. This single-runner guard admits one writer at a time without * holding a lock across the grant/revoke kfuncs. part_pending coalesces - * repartition requests that arrive while it is held. + * refresh and repartition requests that arrive while it is held. * * They live in .bss, not the arena: rr_advance() runs from a bpf_timer * callback, where the verifier rejects atomic ops on arena memory. @@ -1668,33 +1673,46 @@ __noinline void apply_partition(void) } } -/* - * Recompute the split off the node's held caps and apply it. The contexts this - * runs from (the sub-sched and cgroup callbacks, the rr timer) are not - * serialized by the kernel, so a single runner does the work. A caller that - * finds the guard held leaves part_pending set; the holder drains it before - * releasing, with the rr timer as a backstop. +/** + * execute_partition - Run pending partition updates + * + * The rr timer is the backstop if the loop reaches its iteration limit. */ -static void redistribute(void) +static void execute_partition(void) { + u64 pending; s32 i; - __sync_fetch_and_or(&part_pending, 1); + bpf_for(i, 0, 1024) { + if (!part_try_start()) + break; - if (!part_try_start()) - return; + pending = __sync_fetch_and_and(&part_pending, 0); + if (pending & PART_REDISTRIBUTE) { + /* charge elapsed time before repartitioning */ + account_alloc(); + compute_partition(); + apply_partition(); + } else if (pending & PART_REFRESH) { + refresh_usable(); + } - bpf_for(i, 0, 1024) { - __sync_fetch_and_and(&part_pending, 0); - /* charge elapsed time to the current partition before rebuilding it */ - account_alloc(); - compute_partition(); - apply_partition(); + /* + * Requests are published before trying the guard. Releasing it + * before checking pending work ensures a racing request is + * either observed here or handled by a caller that acquires the + * guard. + */ + part_end(); if (!__sync_fetch_and_or(&part_pending, 0)) break; } +} - part_end(); +static void redistribute(void) +{ + __sync_fetch_and_or(&part_pending, PART_REDISTRIBUTE); + execute_partition(); } /* @@ -1708,6 +1726,7 @@ int flush_alloc(void *ctx) if (part_try_start()) { account_alloc(); part_end(); + execute_partition(); } return 0; } @@ -1765,9 +1784,7 @@ static void rr_advance(void) part_end(); - /* a resplit queued while we held the guard supersedes this rotation */ - if (__sync_fetch_and_or(&part_pending, 0)) - redistribute(); + execute_partition(); } /* advance the time-shared cid pool every round_robin_ns */ @@ -2041,20 +2058,8 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) cmask_clear(cid, &qa.avail_cids.mask); } - /* - * When another runner holds the partition guard, set part_pending: - * redistribute() drains it before releasing and rr_advance() checks it - * after, so the deferred refresh lands by the next rr tick. A - * repartition that lost the guard to us runs here. - */ - if (part_try_start()) { - refresh_usable(); - part_end(); - if (__sync_fetch_and_or(&part_pending, 0)) - redistribute(); - } else { - __sync_fetch_and_or(&part_pending, 1); - } + __sync_fetch_and_or(&part_pending, PART_REFRESH); + execute_partition(); } SCX_OPS_CID_DEFINE(qmap_ops, -- knotmirror 0.0.1 From c7a1c6e8004ab12a9c9bfdcb603f60f9bf4a3cee Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Sat, 12 Sep 2026 21:15:18 +0800 Subject: [PATCH 08/18] sched_ext: Close the pre-enable ops error claim window scx_alloc_and_add_sched() publishes ops->priv before scx_root_enable_workfn() switches the state to SCX_ENABLING. An error claimed via scx_bpf_error_bstr() from an associated BPF program in that window is consumed by scx_disable_workfn(), which takes the pre-enable shortcut in scx_root_disable(). The shortcut returns without any teardown and restores SCX_DISABLED with an unconditional scx_set_enable_state() xchg racing the enable workfn's own transition. The enable then completes with the claim consumed: the scheduler stays up but can never be disabled again, and bpf_scx_unreg() frees it while still in use, resulting in a use-after-free. Both WARN_ON_ONCE()s fire back to back: WARNING: kernel/sched/ext/ext.c:7522 at scx_root_enable_workfn+0xeec/0x1be0, CPU#3: scx_enable_help/276 WARNING: kernel/sched/ext/ext.c:6398 at scx_root_disable+0xb50/0xdb8, CPU#0: sched_ext_helpe/664 scx_root_enable_workfn() switches to SCX_ENABLING before the scheduler allocation, so ops->priv is never visible while SCX_DISABLED. The allocation failure path restores SCX_DISABLED. Fixes: 105dcd005be2 ("sched_ext: Introduce scx_prog_sched()") Cc: stable@vger.kernel.org Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index adf5993fa597..83999203a63a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7516,22 +7516,24 @@ static void scx_root_enable_workfn(struct kthread_work *work) #ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp); #endif + /* + * Transition to ENABLING to arm the disable path. Allocation failure + * still unwinds locally. Full disabling on failure applies only after + * scx_alloc_and_add_sched() succeeds. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING); goto err_free_tid_hash; } if (sch->is_cid_type) static_branch_enable(&__scx_is_cid_type); - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) { -- knotmirror 0.0.1 From 9a0b159ff18c8f6fcf982bb81e15a9ceb14db43a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: [PATCH 09/18] sched_ext: scx_qmap: Restore unused idle claims from ops.dispatch() scx_qmap tracks idle cids itself. pick_direct_dispatch_cid() claims a cid by clearing its bit and the task is inserted into that cid's local DSQ, which kicks the CPU. When the task does not arrive, for example because the insert fell back to the global DSQ after an affinity change, the CPU wakes, finds nothing and picks idle again. That is not an idle transition, so ops.update_idle() is not called and the cid stays marked busy until an unrelated task runs on it. Restore the claim from ops.dispatch(). The kick guarantees a dispatch on the kicked CPU, and when it finds nothing to run with a NULL @prev, the CPU is going back to idle. Document the pattern in ops.update_idle(), which reports only actual transitions. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Cc: Andrea Righi --- kernel/sched/ext/internal.h | 6 ++++++ tools/sched_ext/scx_qmap.bpf.c | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 076a351bb3f2..0adaf649d5e0 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -572,6 +572,12 @@ struct sched_ext_ops { * * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle * tracking. + * + * Only actual transitions are reported. A CPU that is claimed with an + * idle pick and kicked but dispatches no task returns to idle without a + * transition. A scheduler tracking idle CPUs itself must restore the + * idle state from ops.dispatch() when it returns without the next task + * to run. */ void (*update_idle)(s32 cpu, bool idle); diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index bda56c37acb5..67b7c01cae55 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -818,10 +818,10 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) batch--; cpuc->dsp_cnt--; if (!batch || !scx_bpf_dispatch_nr_slots()) { - if (scan_shared_dsq(false)) + if (scan_shared_dsq(false) || + scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid))) return; - scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)); - return; + goto prev; } if (!cpuc->dsp_cnt) break; @@ -832,10 +832,14 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) if (scan_shared_dsq(false)) return; - +prev: /* * No other tasks. @prev will keep running. Update its core_sched_seq as * if the task were enqueued and dispatched immediately. + * + * No @prev to keep running means the CPU goes idle. If its claim was + * never used, that is not a transition and ops.update_idle() stays + * silent. Restore the claim here. */ if (prev) { taskc = lookup_task_ctx(prev); @@ -844,6 +848,8 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) taskc->core_sched_seq = qa.core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++; + } else { + cmask_set(cid, &qa.idle_cids.mask); } } -- knotmirror 0.0.1 From a9e3760b0838299649c0d57cca44daaf40ba3c33 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: [PATCH 10/18] sched_ext: Maintain an online cid mask in the scheduler arena Schedulers on the default cid mapping treat [0, nr_online_cids) as the online set and restart on hotplug. Schedulers that install their own mapping with scx_bpf_cid_override() have no way to learn which cids are online: the count no longer identifies members and the CPU-form cpumask is unusable from cid programs. This is an obvious hole in the cid API. Add scx_bpf_online_cmask(), a kernel-maintained cmask in the scheduler's arena, allocated alongside the per-CPU scratch masks and populated after the cid mapping is finalized and before ops.init(), for child schedulers too. The pointer stays valid through ops.exit() with no reference to take. It is the arena offset as a void pointer, the same form struct_ops arena arguments arrive in. The verifier types the void return as a scalar for the program's arena cast. The mask follows the SCX hotplug notifications: seeded from cpu_active_mask and updated before ops.cid_online/offline() runs, so it lags cpu_online_mask only inside a hotplug transition. Updates walk the scheduler list under the lock that also serializes unlinking. Reads are live, not atomic snapshots. Root initialization excludes hotplug. v2: Reworded the getter kerneldoc (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 75 ++++++++++++++++++++++-- kernel/sched/ext/internal.h | 3 +- kernel/sched/ext/sub.c | 10 ++-- tools/sched_ext/include/scx/common.bpf.h | 1 + 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 83999203a63a..70b711c4de6e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3670,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online) s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, lockdep_is_cpus_held()); - if (tbl) + if (tbl) { + struct scx_sched *pos; + cpu_or_cid = tbl[cpu]; + + guard(raw_spinlock_irqsave)(&scx_sched_lock); + list_for_each_entry(pos, &scx_sched_all, all) { + struct scx_cmask *mask = pos->online_cmask; + + if (mask) + __assign_bit(cpu_or_cid, (unsigned long *)mask->bits, + online); + } + } } if (online && SCX_HAS_OP(sch, cpu_online)) @@ -5280,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei); static const char *scx_exit_reason(enum scx_exit_kind kind); static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); + struct scx_cmask *online; + struct scx_cmask_ref ref; int cpu; + /* hotplug stays excluded until the online mask is published */ + lockdep_assert_cpus_held(); + if (!sch->is_cid_type || !sch->arena_pool) return 0; @@ -5301,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) return -ENOMEM; scx_cmask_init(*slot, 0, num_possible_cpus()); } + + /* pack the online mask alongside the scratch masks */ + online = scx_arena_alloc(sch, size); + if (!online) + return -ENOMEM; + + scoped_guard(rcu) { + scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpu_active_mask); + } + sch->online_cmask = online; + return 0; } -static void scx_set_cmask_scratch_free(struct scx_sched *sch) +static void scx_free_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); int cpu; + scx_arena_free(sch, sch->online_cmask, size); if (!sch->set_cmask_scratch) return; @@ -5396,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); - scx_set_cmask_scratch_free(sch); + scx_free_kern_arena_objs(sch); scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); @@ -7601,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } - ret = scx_set_cmask_scratch_alloc(sch); + ret = scx_alloc_kern_arena_objs(sch); if (ret) { cpus_read_unlock(); goto err_disable; @@ -10338,13 +10368,45 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void) * hotplug, which lets schedulers treat [0, nr_online_cids) as the online * range. Schedulers that prefer to handle hotplug without a restart should * install a custom mapping via scx_bpf_cid_override() and track onlining - * through the ops.cid_online / ops.cid_offline callbacks. + * through the ops.cid_online / ops.cid_offline callbacks, starting from the + * mask scx_bpf_online_cmask() returns. */ __bpf_kfunc u32 scx_bpf_nr_online_cids(void) { return num_online_cpus(); } +/** + * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if + * the calling program is not associated with a live cid-form scheduler or the + * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only + * even though arena memory stays writable by the BPF scheduler. The mask + * follows the SCX hotplug notifications: a cid's bit is updated before + * ops.cid_online/offline() runs for it. The pointer is valid from ops.init() + * through ops.exit(). Root ops.init() runs with hotplug excluded. Other + * contexts can observe concurrent updates. + */ +__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_cmask *online; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + online = sch->online_cmask; + if (unlikely(!online)) + return NULL; + + /* BPF rebases by the low 32 bits, like __arena callback args */ + return (void *)((unsigned long)online - sch->arena_kern_base); +} + /** * scx_bpf_this_cid - Return the cid of the CPU this program is running on * @@ -10708,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cids) BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET) BTF_ID_FLAGS(func, scx_bpf_this_cid) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0adaf649d5e0..3464e0f113c1 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1561,6 +1561,7 @@ struct scx_sched { * and passes it to the callback's __arena argument. */ struct scx_cmask * __percpu *set_cmask_scratch; + struct scx_cmask *online_cmask; DECLARE_BITMAP(has_op, SCX_OPI_END); @@ -2087,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); void scx_cgroup_lock(void); void scx_cgroup_unlock(void); #endif -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch); void scx_disable_bypass_dsp(struct scx_sched *sch); void scx_bypass(struct scx_sched *sch, bool bypass); s32 scx_link_sched(struct scx_sched *sch); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 385302d19914..f7aeb1488566 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1805,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto err_disable; } + scoped_guard(cpus_read_lock) { + ret = scx_alloc_kern_arena_objs(sch); + if (ret) + goto err_disable; + } + if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { @@ -1815,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - struct scx_sub_attach_args sub_attach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 76f5e025e107..2ddb01a059fd 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -113,6 +113,7 @@ s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; u32 scx_bpf_nr_cids(void) __ksym __weak; u32 scx_bpf_nr_online_cids(void) __ksym __weak; +const void __arena *scx_bpf_online_cmask(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; -- knotmirror 0.0.1 From 7de9a6fb44eae4f05e68c805d58b9c618815adfa Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 15 Sep 2026 12:34:44 -1000 Subject: [PATCH 11/18] sched_ext: Wait for SCX_OPSS_DISPATCHING before reenqueueing a task ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") moved the final ops_state store in scx_dispatch_enqueue() after the DSQ unlock so that the custody update and ops.dequeue() precede it. A task can thus be found on a DSQ while still SCX_OPSS_DISPATCHING. The dequeue and core-sched pick paths wait for the state to clear in ops_dequeue() but the reenqueue paths don't. A reenqueue in that window runs ops.enqueue() and sets SCX_OPSS_QUEUED before the dispatch has completed. The dispatcher's final store then overwrites it with SCX_OPSS_NONE and finish_dispatch() drops every later dispatch of the task. Wait for SCX_OPSS_DISPATCHING to clear before dequeueing a task for reenqueue, the same way ops_dequeue() does. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.c | 1 + 3 files changed, 15 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 70b711c4de6e..725b890fcab9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4404,6 +4404,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p + * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p + * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + */ +void scx_reenq_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,6 +4458,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4570,6 +4582,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_reenq_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f113c1..115c96fbf932 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2078,6 +2078,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_reenq_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index f7aeb1488566..a30c965b175d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -801,6 +801,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) -- knotmirror 0.0.1 From df5cdc2c832ca4e8a6d774596b9005558761a403 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 16 Sep 2026 12:00:24 -1000 Subject: [PATCH 12/18] sched_ext: Derive SCX_RQ_IN_WAKEUP from the core enqueue flags schedule_deferred_locked() skips scheduling a deferred action while SCX_RQ_IN_WAKEUP is set and relies on the task_woken_scx() call that follows a wakeup enqueue to run it. enqueue_task_scx() sets the flag from the merged enqueue flags, which include the flags stashed for a remote activation. move_remote_task_to_local_dsq() thus sets SCX_RQ_IN_WAKEUP on the destination rq when the moved task was woken up, although no task_woken_scx() follows that activation. An IMMED insert into a busy destination requests a local reenqueue during that enqueue. The request gets linked but not scheduled and stays pending until an unrelated wakeup or preemption on that CPU runs the deferred actions. The IMMED task sits behind the running task in the meantime. If nothing runs them before the scheduler is disabled, the request outlives the scheduler and points into its freed per-cpu area, which the next scheduler dereferences from run_deferred(). Test the core enqueue flags for the wakeup bit. Only the core's wakeup path is followed by task_woken_scx(). Fixes: 57ccf5ccdc56 ("sched_ext: Fix enqueue_task_scx() truncation of upper enqueue flags") Cc: stable@vger.kernel.org # v7.1+ Reported-by: Andrea Righi Link: https://lore.kernel.org/all/20260916145807.3250167-1-arighi@nvidia.com/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 725b890fcab9..e207ccd17164 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2141,7 +2141,12 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* -- knotmirror 0.0.1 From cb86607ada73185f321baa7f9d94a08a3a40bbf5 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Thu, 17 Sep 2026 15:52:41 +0800 Subject: [PATCH 13/18] sched_ext: Don't run ops.dequeue() with a DSQ lock held ops.dequeue() is invoked with the source user DSQ's lock still held on the consume and move paths (scx_consume_dispatch_q(), move_task_between_dsqs()). A BPF scheduler which locks the source user DSQ from ops.dequeue() - e.g. by iterating it with bpf_iter_scx_dsq - self-deadlocks. ops.dequeue() can only call the "any" kfuncs and none of them can lock a builtin DSQ, so the global and bypass paths can't deadlock; however, all DSQ locks share one lockdep class, so iterating any user DSQ from ops.dequeue() on those paths trips the recursion check. Move the invocation after the DSQ unlock on all three paths. SCX_TASK_IN_CUSTODY is cleared under the lock serializing the transfer so that the callback is invoked exactly once. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Acked-by: Andrea Righi Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 44 ++++++++++++++++++------------------- kernel/sched/ext/internal.h | 3 +-- kernel/sched/ext/sub.c | 10 ++++----- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e207ccd17164..f568fd9973f6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1499,27 +1499,22 @@ static inline bool task_scx_migrating(struct task_struct *p) return p->scx.sticky_cpu >= 0; } -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) +/* Must be called under the lock serializing @p's custody transfers. */ +static bool task_leave_custody(struct task_struct *p) { if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + return false; p->scx.flags &= ~SCX_TASK_IN_CUSTODY; + return true; } static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - call_task_dequeue(sch, rq, p, 0); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* * Only local inserts get the wakeup treatment below. Rejects kick the @@ -1705,20 +1700,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, if (is_rq_owned) { rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { + bool call_dequeue = false; + /* * Global and bypass DSQs are terminal - the task leaves the - * scheduler's custody, so ops.dequeue() fires here. It can run + * scheduler's custody, so ops.dequeue() fires. It can run * without @p's rq lock (finish_dispatch() passes the dispatch * rq); that's safe because dequeue_task_scx() waits on * SCX_OPSS_DISPATCHING (see the ops_state note above) and so * can't race it. A non-terminal DSQ keeps the task in custody. + * The custody transfer happens under @dsq->lock so that later + * consumers see the flag clear; the callback runs after + * @dsq->lock is dropped because it may lock a DSQ itself. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); + call_dequeue = task_leave_custody(p); else p->scx.flags |= SCX_TASK_IN_CUSTODY; raw_spin_unlock(&dsq->lock); + + if (call_dequeue && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); } /* @@ -2215,7 +2218,7 @@ retry: /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the + * CPU has already passed task_leave_custody() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, @@ -2263,7 +2266,8 @@ retry: * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ - call_task_dequeue(sch, rq, p, deq_flags); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) @@ -2379,14 +2383,10 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl } void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) + u64 enq_flags, struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ - if (!dsq_is_rq_owned(src_dsq)) - lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -2634,8 +2634,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { scx_task_unlink_from_dsq(p, src_dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); @@ -2685,8 +2685,8 @@ retry: if (rq == task_rq) { scx_task_unlink_from_dsq(p, dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 115c96fbf932..d150de10a5c9 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2065,8 +2065,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq); + u64 enq_flags, struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a30c965b175d..3036190b0662 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -555,8 +555,8 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx.dsq_list.node); scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); scx_rescue_admit(rq, p, slice); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { @@ -572,7 +572,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, - &rq->scx.local_dsq, rq); + rq); } out_arm: scx_rescue_timer_arm(rq); @@ -596,8 +596,8 @@ void scx_rescue_flush(struct rq *rq) /* and flush out all pending ones */ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); } timer_delete(&rq->scx.rescue.timer); -- knotmirror 0.0.1 From 9ec7ba20c97d92fa59b351832aeeb934b3b16177 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Thu, 17 Sep 2026 15:52:42 +0800 Subject: [PATCH 14/18] selftests/sched_ext: Test that ops.dequeue() can iterate the consumed DSQ Add a scheduler whose ops.dequeue() iterates the source user DSQ with bpf_iter_scx_dsq. The iteration takes the DSQ's raw spinlock; on a kernel that runs ops.dequeue() while the consume path still holds that lock, the first task consumed self-deadlocks the CPU with IRQs disabled. The watchdog cannot recover from that state, so on an unfixed kernel this test wedges the system instead of failing cleanly. On a fixed kernel the scheduler runs clean and the test passes. Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + .../selftests/sched_ext/dequeue_iter.bpf.c | 73 +++++++++++++++++ .../selftests/sched_ext/dequeue_iter.c | 79 +++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/dequeue_iter.bpf.c create mode 100644 tools/testing/selftests/sched_ext/dequeue_iter.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 3cfe90e0f34f..49897727f535 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -164,6 +164,7 @@ all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubs auto-test-targets := \ create_dsq \ dequeue \ + dequeue_iter \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c new file mode 100644 index 000000000000..76c2c71a90b6 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ops.dequeue() of this scheduler iterates the user DSQ it consumes + * tasks from with bpf_iter_scx_dsq, which takes the DSQ lock. + * On a kernel that still runs ops.dequeue() with that lock held, the + * iteration self-deadlocks the CPU - this test wedges the system on + * unfixed kernels instead of failing cleanly. + * + * Copyright (c) 2026 fangqiurong + */ + +#include + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +#define TEST_DSQ_ID 1000 + +u64 dq_count; + +s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_iter_init) +{ + return scx_bpf_create_dsq(TEST_DSQ_ID, -1); +} + +s32 BPF_STRUCT_OPS(dequeue_iter_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(dequeue_iter_enqueue, struct task_struct *p, u64 enq_flags) +{ + scx_bpf_dsq_insert(p, TEST_DSQ_ID, SCX_SLICE_DFL, enq_flags); +} + +void BPF_STRUCT_OPS(dequeue_iter_dispatch, s32 cpu, struct task_struct *task) +{ + scx_bpf_dsq_move_to_local(TEST_DSQ_ID, 0); +} + +void BPF_STRUCT_OPS(dequeue_iter_dequeue, struct task_struct *p, u64 deq_flags) +{ + struct bpf_iter_scx_dsq it; + struct task_struct *t; + + if (!bpf_iter_scx_dsq_new(&it, TEST_DSQ_ID, 0)) { + while ((t = bpf_iter_scx_dsq_next(&it))) + ; + } + bpf_iter_scx_dsq_destroy(&it); + + __sync_fetch_and_add(&dq_count, 1); +} + +void BPF_STRUCT_OPS(dequeue_iter_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); + scx_bpf_destroy_dsq(TEST_DSQ_ID); +} + +SEC(".struct_ops.link") +struct sched_ext_ops dequeue_iter_ops = { + .init = (void *)dequeue_iter_init, + .select_cpu = (void *)dequeue_iter_select_cpu, + .enqueue = (void *)dequeue_iter_enqueue, + .dispatch = (void *)dequeue_iter_dispatch, + .dequeue = (void *)dequeue_iter_dequeue, + .exit = (void *)dequeue_iter_exit, + .timeout_ms = 1000U, + .name = "dequeue_iter", +}; diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.c b/tools/testing/selftests/sched_ext/dequeue_iter.c new file mode 100644 index 000000000000..f60711bf3b39 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 fangqiurong + */ +#include +#include +#include +#include +#include "dequeue_iter.bpf.skel.h" +#include "scx_test.h" + +#define DQ_TARGET 10 +#define DQ_DEADLINE_MS 3000 + +static unsigned long long now_ms(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return ts.tv_sec * 1000ULL + ts.tv_nsec / 1000000; +} + +static enum scx_test_status setup(void **ctx) +{ + struct dequeue_iter *skel; + + skel = dequeue_iter__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(dequeue_iter__load(skel), "Failed to load skel"); + + *ctx = skel; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + struct dequeue_iter *skel = ctx; + struct bpf_link *link; + unsigned long long end; + + link = bpf_map__attach_struct_ops(skel->maps.dequeue_iter_ops); + SCX_FAIL_IF(!link, "Failed to attach scheduler"); + + end = now_ms() + DQ_DEADLINE_MS; + while (skel->bss->dq_count < DQ_TARGET && !UEI_EXITED(skel, uei) && + now_ms() < end) + usleep(100); + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + + if (skel->bss->dq_count < DQ_TARGET) { + SCX_ERR("ops.dequeue() fired only %llu times", + (unsigned long long)skel->bss->dq_count); + return SCX_TEST_FAIL; + } + + return SCX_TEST_PASS; +} + +static void cleanup(void *ctx) +{ + struct dequeue_iter *skel = ctx; + + dequeue_iter__destroy(skel); +} + +struct scx_test dequeue_iter = { + .name = "dequeue_iter", + .description = "Verify ops.dequeue() can iterate its source user DSQ", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; +REGISTER_SCX_TEST(&dequeue_iter) -- knotmirror 0.0.1 From 3bd46666cfe51e2bd333fb46a0234694ca594230 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 18 Sep 2026 14:28:37 -1000 Subject: [PATCH 15/18] sched_ext: Pass the initial cmask to cid-form ops.enable() The cid-form API has an obvious hole. A task's cid mask is only visible through ops.set_cmask(), which fires on affinity changes and class switches but not when a task enters a scheduler through fork, sub-sched enable or re-home, and there is no p->cpus_ptr equivalent to fall back on. Schedulers work around it by seeding the mask in ops.init_task() from p->cpus_ptr cid by cid, which is subtly wrong: on sub-sched enable and re-home, an affinity change between init_task() and enable() is delivered to the sched the task is still on, and nothing corrects the new sched's copy afterwards. Fix it by adding struct scx_enable_args to cid-form ops.enable() carrying the task's cmask, built in the per-cpu scratch under the rq lock as the task enters the scheduler, and calling set_cmask() with the same mask right after enable(), ahead of set_weight(). A scheduler can then track affinity in set_cmask() alone, and scx_qmap drops its init_task() seed. set_cmask() no longer fires for a cid-form task before it is enabled, and the class-switch republish in switching_to_scx() is limited to the cpu form. This changes the cid-form ops.enable() signature, which is fine as the cid-form API is still considered unreleased. An args struct rather than a bare cmask argument leaves room for more initial state without another signature change, and the cmask travels as a plain arena address because BTF can't type arena struct members yet. v2: The cmask travels as a u64 arena address, cmask_arena_addr, instead of a kernel-typed pointer, with the typing limitation and the planned typed alias documented (Sashiko review). v3: The initial set_cmask() is delivered before set_weight() so that the mask is in place when weight-dependent state is derived (Andrea Righi). Selftest added. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 92 +++++++++++++++++++++++----------- kernel/sched/ext/internal.h | 56 ++++++++++++++++++--- tools/sched_ext/scx_qmap.bpf.c | 3 -- 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f568fd9973f6..ad391a8cbd05 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -447,37 +447,45 @@ static void switch_rq_lock(struct rq *from, struct rq *to) DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); /** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask + * @sch: scx_sched whose scratch to fill + * @cpumask: cpus to translate into cids + * + * The scratch lives in BPF-writable arena memory and its header can't be + * trusted, so it is rewritten from kernel geometry rather than read. Caller + * must hold an rq lock so this cpu is the sole kernel writer for as long as the + * returned address is in use. + */ +static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, + const struct cpumask *cpumask) +{ + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + return kern_va; +} + +/** + * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch + * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - struct scx_cmask_ref ref; - - /* - * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header. set_cmask()'s __arena - * argument takes the kernel address and the struct_ops - * trampoline rebases it into BPF's arena pointer form. The rq - * lock makes this cpu the sole kernel writer. - */ - scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); - scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); - } else { + if (scx_is_cid_type()) + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_fill_cmask_scratch(sch, cpumask)); + else SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); - } } enum scx_dsq_iter_flags { @@ -3634,8 +3642,12 @@ static void set_cpus_allowed_scx(struct task_struct *p, * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. + * + * The cid form receives the initial mask when the task is enabled and + * hears about changes only afterwards, see struct scx_enable_args. */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (SCX_HAS_OP(sch, set_cpumask) && + (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3944,8 +3956,27 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) p->scx.weight = sched_weight_to_cgroup(weight); - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); + if (SCX_HAS_OP(sch, enable)) { + if (scx_is_cid_type()) { + struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); + struct scx_enable_args args = { + .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), + }; + + SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); + } else { + SCX_CALL_OP_TASK(sch, enable, rq, p); + } + } + + /* + * The initial mask also goes out through set_cmask() so a scheduler can + * track affinity there alone, and before set_weight() so that the mask + * is in place when weight-dependent state is derived, see struct + * scx_enable_args. + */ + if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); @@ -4288,9 +4319,10 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) /* * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. + * different scheduler class. Keep the BPF scheduler up-to-date. The cid + * form gets its mask from scx_enable_task(). */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } @@ -8374,10 +8406,11 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused. Some need - * fresh stubs, set_cmask due to an argument type difference and the sub-sched - * notifiers because no cpu-form stub exists to reuse. + * fresh stubs, set_cmask and enable due to argument differences and the + * sub-sched notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} @@ -8398,7 +8431,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, + .enable = sched_ext_ops_cid__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cpuctl_init = sched_ext_ops__cgroup_init, @@ -10421,8 +10454,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) if (unlikely(!online)) return NULL; - /* BPF rebases by the low 32 bits, like __arena callback args */ - return (void *)((unsigned long)online - sch->arena_kern_base); + return (void *)scx_kaddr_to_arena(sch, online); } /** diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d150de10a5c9..1df8f583b0ec 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -250,6 +250,31 @@ struct scx_exit_task_args { bool cancelled; }; +/** + * struct scx_enable_args - Argument container for cid-form ops.enable() + * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on + * + * @cmask_arena_addr is the task's affinity as it enters the scheduler. + * set_cmask() delivers the same mask right after enable(), before set_weight() + * and the first enqueue, then every affinity change afterwards, and is never + * called before enable(). A scheduler may therefore track affinity in + * set_cmask() alone. + * + * The kernel builds the mask in the scheduler arena from its own geometry, so + * the header is valid regardless of what the scheduler last wrote there. The + * memory is per-cpu scratch reused once the callback returns: copy the bits + * out, don't keep the address. The set_cmask() argument follows the same rules. + * + * The address is a plain value rather than a typed pointer because BTF can't + * mark a struct member as an arena pointer yet and a pointer member would reach + * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena * + * before use. Once arena members can be typed, a typed alias will join this + * field in an anonymous union at the same offset. + */ +struct scx_enable_args { + u64 cmask_arena_addr; +}; + /* argument container for ops.cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ @@ -1037,6 +1062,7 @@ struct sched_ext_ops { * - dispatch -> dispatch (cpu arg is now cid) * - update_idle -> update_idle (cpu arg is now cid) * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - enable -> enable (takes struct scx_enable_args) * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid @@ -1070,7 +1096,7 @@ struct sched_ext_ops_cid { struct scx_init_task_args *args); void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); + void (*enable)(struct task_struct *p, struct scx_enable_args *args); void (*disable)(struct task_struct *p); void (*dump)(struct scx_dump_ctx *ctx); void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); @@ -1533,7 +1559,8 @@ struct scx_sched { * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel * access either view of the same storage without function-pointer * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + * callbacks whose cid-form signature differs (set_cmask, enable, + * select_cid, cid_online, ...). */ union { struct sched_ext_ops ops; @@ -1556,9 +1583,9 @@ struct scx_sched { uintptr_t arena_kern_base; /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and passes it to the callback's __arena argument. + * Per-CPU arena cmask the kernel fills from a task's cpumask and hands + * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are + * the kernel addresses. */ struct scx_cmask * __percpu *set_cmask_scratch; struct scx_cmask *online_cmask; @@ -1669,6 +1696,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } +/** + * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel address inside @sch's arena + * + * __arena callback arguments need no translation. Addresses handed to BPF any + * other way, such as struct fields and kfunc return values, go through this. + */ +static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (uintptr_t)kaddr - sch->arena_kern_base; +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -2302,9 +2342,9 @@ do { \ } while (0) /* - * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs - * this: it takes an arena cmask address instead of a cpumask, so it cannot be - * invoked via its cpu-form set_cpumask() slot. + * Dispatch a task op through the cid-form ops_cid table, for the ops whose + * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena + * cmask instead of a cpumask and enable() takes scx_enable_args. */ #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 67b7c01cae55..2f3653199842 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -961,9 +961,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_task, struct task_struct *p, taskc->highpri = false; taskc->core_sched_seq = 0; cmask_init(&taskc->cpus_allowed, 0, scx_bpf_nr_cids()); - bpf_rcu_read_lock(); - cmask_from_cpumask(&taskc->cpus_allowed, p->cpus_ptr); - bpf_rcu_read_unlock(); v = bpf_task_storage_get(&task_ctx_stor, p, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE); -- knotmirror 0.0.1 From d781d1b78acf547bafeb1592e8ba4020dce515c6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 19 Sep 2026 04:08:20 -1000 Subject: [PATCH 16/18] selftests/sched_ext: Check the cmask cid-form ops.enable() receives cid-form ops.enable() now hands the task's cmask to the scheduler and set_cmask() repeats it right after. Add a cid-form selftest that checks both against p->cpus_ptr, that they match each other, that the initial set_cmask() lands before set_weight() and before the task first becomes runnable, and that set_cmask() never precedes enable(), across class-switch enables, fork-path enables and live affinity changes. v2: Mismatch details returned through a caller-local struct instead of globals, alloc_words validated in the header check, loop bounded by nr_cids directly (Andrea Righi). Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + .../selftests/sched_ext/enable_cmask.bpf.c | 217 ++++++++++++++++++ .../selftests/sched_ext/enable_cmask.c | 138 +++++++++++ 3 files changed, 356 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/enable_cmask.bpf.c create mode 100644 tools/testing/selftests/sched_ext/enable_cmask.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 49897727f535..4e06d0baaeec 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -169,6 +169,7 @@ auto-test-targets := \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ dsp_local_on \ + enable_cmask \ enq_select_cpu \ exit \ hotplug \ diff --git a/tools/testing/selftests/sched_ext/enable_cmask.bpf.c b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c new file mode 100644 index 000000000000..0068f3c7ab3c --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A cid-form scheduler checking the cmask cid-form ops.enable() receives: the + * header, every cid bit against p->cpus_ptr, and that set_cmask() follows with + * the same mask before set_weight() and before the task first becomes runnable, + * and never runs before enable(). + * + * Copyright (c) 2026 Tejun Heo + */ +#include + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1 << 16); +} arena SEC(".maps"); + +struct task_ctx { + u64 enable_fp; /* fingerprint of the mask enable() received */ + bool enabled; + bool pending; /* enable() ran, the initial set_cmask() hasn't */ +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +/* details of a cid bit mismatch, filled by check_mask() */ +struct mask_mismatch { + s32 cid; + bool want; + bool got; +}; + +u64 nr_enable, nr_initial_set_cmask, nr_set_cmask, nr_set_weight; + +UEI_DEFINE(uei); + +static struct task_ctx *lookup_task_ctx(struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) + scx_bpf_error("task_ctx lookup failed for %s[%d]", p->comm, p->pid); + return tctx; +} + +/* + * Verify @m's header and every cid bit against @p's cpumask and fingerprint the + * bits into @fp. Return 0 on success, -EINVAL on a bad header, -ENOENT on a cid + * without a cpu and -EIO on a bit mismatch with the details in @mm. + */ +static int check_mask(struct task_struct *p, const struct scx_cmask __arena *m, u64 *fp, + struct mask_mismatch *mm) +{ + u32 nr_cids = scx_bpf_nr_cids(); + u64 h = 0; + s32 cid; + + if (m->base || m->nr_cids != nr_cids || m->alloc_words != CMASK_NR_WORDS(nr_cids)) + return -EINVAL; + + bpf_for(cid, 0, nr_cids) { + bool want, got; + s32 cpu; + + cpu = scx_bpf_cid_to_cpu(cid); + if (cpu < 0) + return -ENOENT; + want = bpf_cpumask_test_cpu(cpu, p->cpus_ptr); + got = cmask_test(cid, m); + if (want != got) { + mm->cid = cid; + mm->want = want; + mm->got = got; + return -EIO; + } + h = h * 31 + got; + } + + *fp = h; + return 0; +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enable_cmask_init_task, struct task_struct *p, + struct scx_init_task_args *args) +{ + if (!bpf_task_storage_get(&task_ctx_stor, p, 0, BPF_LOCAL_STORAGE_GET_F_CREATE)) + return -ENOMEM; + return 0; +} + +void BPF_STRUCT_OPS(enable_cmask_enable, struct task_struct *p, struct scx_enable_args *args) +{ + struct scx_cmask __arena *m = (struct scx_cmask __arena *)args->cmask_arena_addr; + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_enable, 1); + if (tctx->enabled || tctx->pending) { + scx_bpf_error("enable: %s[%d] enabled twice", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &tctx->enable_fp, &mm); + if (ret) { + scx_bpf_error("enable: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + tctx->enabled = true; + tctx->pending = true; +} + +void BPF_STRUCT_OPS(enable_cmask_set_cmask, struct task_struct *p, + struct scx_cmask __arena *m) +{ + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + u64 fp; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_cmask, 1); + if (!tctx->enabled) { + scx_bpf_error("set_cmask: %s[%d] not enabled", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &fp, &mm); + if (ret) { + scx_bpf_error("set_cmask: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + + if (tctx->pending) { + if (fp != tctx->enable_fp) { + scx_bpf_error("set_cmask: %s[%d] initial mask differs from enable()", + p->comm, p->pid); + return; + } + tctx->pending = false; + __sync_fetch_and_add(&nr_initial_set_cmask, 1); + } +} + +void BPF_STRUCT_OPS(enable_cmask_set_weight, struct task_struct *p, u32 weight) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_weight, 1); + if (tctx->pending) + scx_bpf_error("set_weight: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_runnable, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + if (tctx->pending) + scx_bpf_error("runnable: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_disable, struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + tctx->enabled = false; + tctx->pending = false; +} + +void BPF_STRUCT_OPS(enable_cmask_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SCX_OPS_CID_DEFINE(enable_cmask_ops, + .init_task = (void *)enable_cmask_init_task, + .enable = (void *)enable_cmask_enable, + .set_cmask = (void *)enable_cmask_set_cmask, + .set_weight = (void *)enable_cmask_set_weight, + .runnable = (void *)enable_cmask_runnable, + .disable = (void *)enable_cmask_disable, + .exit = (void *)enable_cmask_exit, + .flags = SCX_OPS_SWITCH_PARTIAL, + .name = "enable_cmask"); diff --git a/tools/testing/selftests/sched_ext/enable_cmask.c b/tools/testing/selftests/sched_ext/enable_cmask.c new file mode 100644 index 000000000000..556bcad4431d --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Tejun Heo */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "enable_cmask.bpf.skel.h" +#include "scx_test.h" + +#define SCHED_EXT 7 +#define NR_CHILDREN 8 +#define MAX_CPUS 1024 + +static int cpus[MAX_CPUS]; +static int nr_cpus; + +static void spin_ms(int ms) +{ + struct timespec start, now; + + clock_gettime(CLOCK_MONOTONIC, &start); + do { + clock_gettime(CLOCK_MONOTONIC, &now); + } while ((now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000 < ms); +} + +static int pin(pid_t pid, int idx) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpus[idx % nr_cpus], &set); + return sched_setaffinity(pid, sizeof(set), &set); +} + +/* + * Pin, switch to SCHED_EXT for a class-switch enable, fork a grandchild that + * inherits the policy for a fork-path enable, then change affinity a few times + * while running for set_cmask() on live tasks. + */ +static int child(int idx) +{ + struct sched_param param = {}; + int i, status; + pid_t pid; + + if (pin(0, idx) || sched_setscheduler(0, SCHED_EXT, ¶m)) + return 1; + + pid = fork(); + if (pid < 0) + return 1; + if (!pid) { + spin_ms(20); + return 0; + } + + for (i = 1; i <= 4; i++) { + if (pin(0, idx + i)) + return 1; + spin_ms(5); + } + + return waitpid(pid, &status, 0) == pid && !status ? 0 : 1; +} + +static enum scx_test_status run(void *ctx) +{ + struct enable_cmask *skel; + struct bpf_link *link; + pid_t pids[NR_CHILDREN]; + cpu_set_t set; + int i, status, failed = 0; + + if (!__COMPAT_struct_has_field("scx_enable_args", "cmask_arena_addr")) + return SCX_TEST_SKIP; + + SCX_FAIL_IF(sched_getaffinity(0, sizeof(set), &set), "Failed to read affinity"); + for (i = 0; i < MAX_CPUS && i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &set)) + cpus[nr_cpus++] = i; + if (nr_cpus < 2) + return SCX_TEST_SKIP; + + skel = enable_cmask__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(enable_cmask__load(skel), "Failed to load skel"); + + link = bpf_map__attach_struct_ops(skel->maps.enable_cmask_ops); + SCX_FAIL_IF(!link, "Failed to attach struct_ops"); + + for (i = 0; i < NR_CHILDREN; i++) { + pids[i] = fork(); + SCX_FAIL_IF(pids[i] < 0, "Failed to fork"); + if (!pids[i]) + exit(child(i)); + } + + /* affinity changes from the outside race with the children's own */ + for (i = 0; i < NR_CHILDREN; i++) + pin(pids[i], i + NR_CHILDREN); + + for (i = 0; i < NR_CHILDREN; i++) { + if (waitpid(pids[i], &status, 0) != pids[i] || status) + failed++; + } + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + SCX_EQ(failed, 0); + SCX_GE(skel->bss->nr_enable, 2 * NR_CHILDREN); + SCX_EQ(skel->bss->nr_initial_set_cmask, skel->bss->nr_enable); + SCX_GT(skel->bss->nr_set_cmask, skel->bss->nr_initial_set_cmask); + SCX_GE(skel->bss->nr_set_weight, skel->bss->nr_enable); + printf("enable=%lu initial_set_cmask=%lu set_cmask=%lu set_weight=%lu\n", + (unsigned long)skel->bss->nr_enable, + (unsigned long)skel->bss->nr_initial_set_cmask, + (unsigned long)skel->bss->nr_set_cmask, + (unsigned long)skel->bss->nr_set_weight); + + enable_cmask__destroy(skel); + return SCX_TEST_PASS; +} + +struct scx_test enable_cmask = { + .name = "enable_cmask", + .description = "Check the cid-form ops.enable() cmask and the set_cmask() after it", + .run = run, +}; +REGISTER_SCX_TEST(&enable_cmask) -- knotmirror 0.0.1 From 4409a85735cdca5c4c400c0dad1feee091872eee Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Wed, 23 Sep 2026 19:07:37 +0800 Subject: [PATCH 17/18] sched_ext: Count SCX_EV_SUB_BYPASS_DISPATCH in the dispatch fallback When a descendant scheduler enters bypass mode, its tasks are parked in the bypass DSQs of the nearest non-bypassing ancestor, which is then responsible for running them. On behalf of such a non-bypassing host, scx_dispatch_sched() consumes those bypass DSQs from two places: the attempt made every SCX_BYPASS_HOST_NTH dispatches, and the end-of-dispatch fallback that keeps the CPU from going idle while bypassed descendants still have tasks queued. The former increments SCX_EV_SUB_BYPASS_DISPATCH but the latter does not, even though both perform the same scx_consume_dispatch_q() on the same bypass DSQ. The descendant bypass dispatches done by the fallback are therefore missing from the counter exposed via sysfs, scx_dump_state() and the scx_bpf_events() kfunc, which under-reports the actual number of such dispatches. Add the missing __scx_add_event() so the fallback counts them too. When @sch itself is bypassing, scx_dispatch_sched() takes the earlier self-bypass branch and returns before reaching these host paths; that mode is accounted for by SCX_EV_BYPASS_DISPATCH at enqueue time and is intentionally left unchanged. Fixes: 025b1bd41965 ("sched_ext: Implement hierarchical bypass mode") Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/inlines.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index ed423bcc26b8..2ff5479334cb 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -129,8 +129,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; + } return SCX_DSP_NONE; } -- knotmirror 0.0.1 From 94480606a677deb68d5622cf0ded88626514b3f2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 26 Sep 2026 09:23:02 -1000 Subject: [PATCH 18/18] sched_ext: Add a size argument to scx_bpf_cid_topo() so struct scx_cid_topo can grow scx_bpf_cid_topo() copies struct scx_cid_topo into a buffer the BPF program sized from its own vmlinux.h while the verifier sizes the write from the running kernel's BTF. The struct may grow and each growth then breaks every scheduler built against the older layout, rejected at load or written past its buffer. This is the usual hole for a struct handed to BPF, closed elsewhere with a size argument, and it was missed here. Take the buffer size, copy the smaller of it and the kernel's struct and set the rest to -1. Accesses to the copy are CO-RE relocated, so the struct can grow by appending fields, which its comment now states. The kfunc changes in place: the cid interface is still being finalized and no released scheduler uses the current form. Fixes: e9b55af47edf ("sched_ext: Add topological CPU IDs (cids)") Cc: stable@vger.kernel.org # v7.2+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 26 +++++++++++++++--------- kernel/sched/ext/types.h | 4 ++++ tools/sched_ext/include/scx/common.bpf.h | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 39f88deb94bc..d4d5f537832a 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -912,30 +912,36 @@ bool scx_cmask_empty(const struct scx_cmask *m) /** * scx_bpf_cid_topo - Copy out per-cid topology info * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call + * @out: where to copy the topology info + * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo) * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. All fields are also set to -1 when no cid tables have been - * published yet, which a program may observe while racing the root enable. + * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out + * of range. If @cid is valid but in the no-topo section, all fields are set to + * -1. All fields are also set to -1 when no cid tables have been published yet, + * which a program may observe while racing the root enable. + * + * The program's struct may be older or newer than the kernel's. The smaller of + * @out__sz and the kernel's size is copied and the rest of @out is set to -1. */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz, const struct bpf_prog_aux *aux) { + size_t len = min(out__sz, sizeof(*out)); struct scx_cid_topo *topo; struct scx_sched *sch; + /* the error cases and fields the kernel lacks read as -1 */ + memset(out, 0xff, out__sz); + guard(rcu)(); sch = scx_prog_sched(aux); topo = rcu_dereference(scx_cid_topo); - if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { - *out__uninit = SCX_CID_TOPO_NEG; + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) return; - } - *out__uninit = topo[cid]; + memcpy(out, &topo[cid], len); } __bpf_kfunc_end_defs(); diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 943d8d429a2c..139176cf9fc6 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -70,6 +70,10 @@ enum scx_consts { * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * New fields are appended, never inserted: scx_bpf_cid_topo() copies this + * struct out sized by the program's own layout, and an older program's copy + * must stay a prefix of the kernel's. + * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) * @llc_cid: first cid of this cid's LLC diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 2ddb01a059fd..22f24ebef8a9 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -106,7 +106,7 @@ u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; -void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; +void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; -- knotmirror 0.0.1