Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271// SPDX-License-Identifier: GPL-2.0/* * Verify that ops.dequeue() is called when a task leaves the BPF scheduler's * custody by being moved to the local DSQ of a CPU other than the one whose * rq it's on (move_remote_task_to_local_dsq()). * * ops.enqueue() puts every task into custody and ops.dispatch() moves it to * the dispatching CPU's local DSQ, so most moves cross CPUs. With * @test_use_move_to_local, tasks are queued on a user DSQ and consumed with * scx_bpf_dsq_move_to_local(). Otherwise, they are queued in a BPF queue and * dispatched with SCX_DSQ_LOCAL_ON. * * Copyright (c) 2026 Google LLC. */
#include <scx/common.bpf.h>
#define SHARED_DSQ 0#define MAX_DISPATCH_POPS 8
char _license[] SEC("license") = "GPL";
UEI_DEFINE(uei);
struct { __uint(type, BPF_MAP_TYPE_QUEUE); __uint(max_entries, 32768); __type(value, s32);} global_queue SEC(".maps");
enum task_state { TASK_NONE = 0, TASK_ENQUEUED, /* in BPF custody, waiting for ops.dequeue() */ TASK_DISPATCHED, /* left custody */};
struct task_ctx { enum task_state state; s32 enq_cpu; /* scx_bpf_task_cpu() at ops.enqueue() */ u64 enqueue_seq;};
struct { __uint(type, BPF_MAP_TYPE_TASK_STORAGE); __uint(map_flags, BPF_F_NO_PREALLOC); __type(key, int); __type(value, struct task_ctx);} task_ctx_stor SEC(".maps");
/* core_cookie only exists with CONFIG_SCHED_CORE */struct task_struct___core_sched { unsigned long core_cookie;} __attribute__((preserve_access_index));
bool test_use_move_to_local;
u64 enqueue_cnt, dequeue_cnt, dispatch_dequeue_cnt, change_dequeue_cnt;u64 remote_dispatch_cnt, remote_running_cnt, missed_dequeue_cnt;u64 core_sched_exec_dequeue_cnt;
static struct task_ctx *lookup_task_ctx(struct task_struct *p){ return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);}
static bool task_has_core_cookie(struct task_struct *p){ struct task_struct___core_sched *t = (void *)p;
if (!bpf_core_field_exists(t->core_cookie)) return false; return BPF_CORE_READ(t, core_cookie);}
s32 BPF_STRUCT_OPS(dequeue_remote_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags){ /* no direct dispatch, always go through ops.enqueue() */ return prev_cpu;}
void BPF_STRUCT_OPS(dequeue_remote_enqueue, struct task_struct *p, u64 enq_flags){ struct task_ctx *tctx; s32 pid = p->pid;
tctx = lookup_task_ctx(p); if (!tctx) { scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); return; }
/* the previous custody period must have ended with ops.dequeue() */ if (tctx->state == TASK_ENQUEUED) scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu", p->pid, p->comm, tctx->enqueue_seq);
/* * Mark @p as enqueued before making it visible to ops.dispatch() on * other CPUs, which skips queue entries of tasks not in ENQUEUED * state as stale. */ tctx->state = TASK_ENQUEUED; tctx->enq_cpu = scx_bpf_task_cpu(p); tctx->enqueue_seq++;
if (test_use_move_to_local) { scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags); } else if (bpf_map_push_elem(&global_queue, &pid, 0)) { scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); tctx->state = TASK_DISPATCHED; tctx->enq_cpu = -1; goto out; }
__sync_fetch_and_add(&enqueue_cnt, 1);out: scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE);}
void BPF_STRUCT_OPS(dequeue_remote_dequeue, struct task_struct *p, u64 deq_flags){ struct task_ctx *tctx;
__sync_fetch_and_add(&dequeue_cnt, 1);
tctx = lookup_task_ctx(p); if (!tctx) return;
/* * Only core scheduling can pick a task straight out of custody, and * only if the task has a core cookie. Otherwise, the custody exit was * missed when @p was inserted into a local DSQ and got deferred until * @p was picked. */ if ((deq_flags & SCX_DEQ_CORE_SCHED_EXEC) && !task_has_core_cookie(p)) { __sync_fetch_and_add(&core_sched_exec_dequeue_cnt, 1); scx_bpf_error("%d (%s): late ops.dequeue() with SCX_DEQ_CORE_SCHED_EXEC (enq_cpu=%d cpu=%d seq=%llu)", p->pid, p->comm, tctx->enq_cpu, scx_bpf_task_cpu(p), tctx->enqueue_seq); }
/* ops.dequeue() ends the custody period started by ops.enqueue() */ if (tctx->state != TASK_ENQUEUED) scx_bpf_error("%d (%s): dequeue outside custody deq_flags=0x%llx state=%d seq=%llu", p->pid, p->comm, deq_flags, tctx->state, tctx->enqueue_seq);
if (deq_flags & SCX_DEQ_SCHED_CHANGE) { __sync_fetch_and_add(&change_dequeue_cnt, 1); tctx->state = TASK_NONE; } else { __sync_fetch_and_add(&dispatch_dequeue_cnt, 1); tctx->state = TASK_DISPATCHED; }}
void BPF_STRUCT_OPS(dequeue_remote_dispatch, s32 cpu, struct task_struct *prev){ struct task_ctx *tctx; struct task_struct *p; s32 pid; int i;
if (test_use_move_to_local) { scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); return; }
/* pop past stale entries so that they don't leave this CPU idle */ bpf_for(i, 0, MAX_DISPATCH_POPS) { if (bpf_map_pop_elem(&global_queue, &pid)) return;
p = bpf_task_from_pid(pid); if (!p) continue;
/* * Entries are stale if @p left custody through a property * change dequeue or was dispatched from a duplicate entry. */ tctx = lookup_task_ctx(p); if (!tctx || tctx->state != TASK_ENQUEUED) { bpf_task_release(p); continue; }
if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { if (scx_bpf_task_cpu(p) != cpu) __sync_fetch_and_add(&remote_dispatch_cnt, 1); scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_DFL, 0); } else { scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); }
bpf_task_release(p); return; }
/* out of pops with entries left, retry instead of idling this CPU */ if (!bpf_map_peek_elem(&global_queue, &pid)) scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE);}
void BPF_STRUCT_OPS(dequeue_remote_running, struct task_struct *p){ struct task_ctx *tctx;
tctx = lookup_task_ctx(p); if (!tctx) return;
/* tasks can only run from a local DSQ, i.e. after leaving custody */ if (tctx->state == TASK_ENQUEUED) { __sync_fetch_and_add(&missed_dequeue_cnt, 1); scx_bpf_error("%d (%s): running without ops.dequeue() (enq_cpu=%d cpu=%d seq=%llu)", p->pid, p->comm, tctx->enq_cpu, scx_bpf_task_cpu(p), tctx->enqueue_seq); return; }
if (tctx->enq_cpu >= 0 && tctx->enq_cpu != scx_bpf_task_cpu(p)) __sync_fetch_and_add(&remote_running_cnt, 1); tctx->enq_cpu = -1;}
s32 BPF_STRUCT_OPS(dequeue_remote_init_task, struct task_struct *p, struct scx_init_task_args *args){ struct task_ctx *tctx;
tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (!tctx) return -ENOMEM;
/* task storage persists across attachments, start from scratch */ tctx->state = TASK_NONE; tctx->enq_cpu = -1; tctx->enqueue_seq = 0;
return 0;}
s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_remote_init){ return scx_bpf_create_dsq(SHARED_DSQ, -1);}
void BPF_STRUCT_OPS(dequeue_remote_exit, struct scx_exit_info *ei){ UEI_RECORD(uei, ei);}
SEC(".struct_ops.link")struct sched_ext_ops dequeue_remote_ops = { .select_cpu = (void *)dequeue_remote_select_cpu, .enqueue = (void *)dequeue_remote_enqueue, .dequeue = (void *)dequeue_remote_dequeue, .dispatch = (void *)dequeue_remote_dispatch, .running = (void *)dequeue_remote_running, .init_task = (void *)dequeue_remote_init_task, .init = (void *)dequeue_remote_init, .exit = (void *)dequeue_remote_exit, .flags = SCX_OPS_ENQ_LAST, .name = "dequeue_remote",};