diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1bd1b461ff30..3ef4147c2d0a 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -675,7 +675,7 @@ void scx_rescue_init(struct rq *rq) } /** - * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -695,12 +695,9 @@ void scx_rescue_init(struct rq *rq) * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { - if (!scx_has_subs()) - return &rq->scx.local_dsq; - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -783,14 +780,12 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. * Rejection can't happen for root. */ -void scx_reenq_reject(struct rq *rq) +void __scx_reenq_reject(struct rq *rq) { LIST_HEAD(tasks); struct task_struct *p, *n; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* @@ -925,7 +920,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */ @@ -950,7 +945,7 @@ static void discard_queued_syncs(struct rq *rq) } /** - * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * @@ -962,16 +957,14 @@ static void discard_queued_syncs(struct rq *rq) * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..6a7076b9a1dd 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -28,16 +28,16 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void scx_reenq_reject(struct rq *rq); +void __scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -72,6 +72,33 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_process_sync_ecaps(rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, + struct rq *rq, + struct task_struct *p, + u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + return __scx_resolve_local_dsq(sch, rq, p, enq_flags); +} + +static inline void scx_reenq_reject(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_reenq_reject(rq); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }