Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

Merge branch 'sched/fast-headers' into sched/core

Merge the scheduler build speedup of the fast-headers tree.

Cumulative scheduler (kernel/sched/) build time speedup on a
Linux distribution's config, which enables all scheduler features,
compared to the vanilla kernel:

_____________________________________________________________________________
|
| Vanilla kernel (v5.13-rc7):
|_____________________________________________________________________________
|
| Performance counter stats for 'make -j96 kernel/sched/' (3 runs):
|
| 126,975,564,374 instructions # 1.45 insn per cycle ( +- 0.00% )
| 87,637,847,671 cycles # 3.959 GHz ( +- 0.30% )
| 22,136.96 msec cpu-clock # 7.499 CPUs utilized ( +- 0.29% )
|
| 2.9520 +- 0.0169 seconds time elapsed ( +- 0.57% )
|_____________________________________________________________________________
|
| Patched kernel:
|_____________________________________________________________________________
|
| Performance counter stats for 'make -j96 kernel/sched/' (3 runs):
|
| 50,420,496,914 instructions # 1.47 insn per cycle ( +- 0.00% )
| 34,234,322,038 cycles # 3.946 GHz ( +- 0.31% )
| 8,675.81 msec cpu-clock # 3.053 CPUs utilized ( +- 0.45% )
|
| 2.8420 +- 0.0181 seconds time elapsed ( +- 0.64% )
|_____________________________________________________________________________

Summary:

- CPU time used to build the scheduler dropped by -60.9%, a reduction
from 22.1 clock-seconds to 8.7 clock-seconds.

- Wall-clock time to build the scheduler dropped by -3.9%, a reduction
from 2.95 seconds to 2.84 seconds.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

+407 -163
+1
arch/arm64/include/asm/paravirt_api_clock.h
··· 1 + #include <asm/paravirt.h>
+1
arch/x86/include/asm/paravirt_api_clock.h
··· 1 + #include <asm/paravirt.h>
+1
include/linux/cgroup_api.h
··· 1 + #include <linux/cgroup.h>
+1
include/linux/cpumask_api.h
··· 1 + #include <linux/cpumask.h>
+1
include/linux/fs_api.h
··· 1 + #include <linux/fs.h>
+1
include/linux/gfp_api.h
··· 1 + #include <linux/gfp.h>
+1
include/linux/hashtable_api.h
··· 1 + #include <linux/hashtable.h>
+1
include/linux/hrtimer_api.h
··· 1 + #include <linux/hrtimer.h>
+1
include/linux/kobject_api.h
··· 1 + #include <linux/kobject.h>
+1
include/linux/kref_api.h
··· 1 + #include <linux/kref.h>
+1
include/linux/ktime_api.h
··· 1 + #include <linux/ktime.h>
+1
include/linux/llist_api.h
··· 1 + #include <linux/llist.h>
+1
include/linux/lockdep_api.h
··· 1 + #include <linux/lockdep.h>
+1
include/linux/mm_api.h
··· 1 + #include <linux/mm.h>
+1
include/linux/mutex_api.h
··· 1 + #include <linux/mutex.h>
+1
include/linux/perf_event_api.h
··· 1 + #include <linux/perf_event.h>
+1
include/linux/pgtable_api.h
··· 1 + #include <linux/pgtable.h>
+1
include/linux/ptrace_api.h
··· 1 + #include <linux/ptrace.h>
+1
include/linux/rcuwait_api.h
··· 1 + #include <linux/rcuwait.h>
+1
include/linux/refcount_api.h
··· 1 + #include <linux/refcount.h>
+1
include/linux/sched/affinity.h
··· 1 + #include <linux/sched.h>
+1
include/linux/sched/cond_resched.h
··· 1 + #include <linux/sched.h>
+2
include/linux/sched/deadline.h
··· 6 6 * NORMAL/BATCH tasks. 7 7 */ 8 8 9 + #include <linux/sched.h> 10 + 9 11 #define MAX_DL_PRIO 0 10 12 11 13 static inline int dl_prio(int prio)
+1
include/linux/sched/posix-timers.h
··· 1 + #include <linux/posix-timers.h>
+1
include/linux/sched/rseq_api.h
··· 1 + #include <linux/rseq.h>
+1
include/linux/sched/task_flags.h
··· 1 + #include <linux/sched.h>
+1
include/linux/sched/thread_info_api.h
··· 1 + #include <linux/thread_info.h>
+2
include/linux/sched_clock.h
··· 5 5 #ifndef LINUX_SCHED_CLOCK 6 6 #define LINUX_SCHED_CLOCK 7 7 8 + #include <linux/types.h> 9 + 8 10 #ifdef CONFIG_GENERIC_SCHED_CLOCK 9 11 /** 10 12 * struct clock_read_data - data required to read from sched_clock()
+1
include/linux/seqlock_api.h
··· 1 + #include <linux/seqlock.h>
+1
include/linux/softirq.h
··· 1 + #include <linux/interrupt.h>
+1
include/linux/spinlock_api.h
··· 1 + #include <linux/spinlock.h>
+1
include/linux/swait_api.h
··· 1 + #include <linux/swait.h>
+1
include/linux/syscalls_api.h
··· 1 + #include <linux/syscalls.h>
+1
include/linux/u64_stats_sync_api.h
··· 1 + #include <linux/u64_stats_sync.h>
+1
include/linux/wait_api.h
··· 1 + #include <linux/wait.h>
+1
include/linux/workqueue_api.h
··· 1 + #include <linux/workqueue.h>
+10 -18
kernel/sched/Makefile
··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 - ifdef CONFIG_FUNCTION_TRACER 3 - CFLAGS_REMOVE_clock.o = $(CC_FLAGS_FTRACE) 4 - endif 5 2 6 3 # The compilers are complaining about unused variables inside an if(0) scope 7 4 # block. This is daft, shut them up. ··· 22 25 CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer 23 26 endif 24 27 25 - obj-y += core.o loadavg.o clock.o cputime.o 26 - obj-y += idle.o fair.o rt.o deadline.o 27 - obj-y += wait.o wait_bit.o swait.o completion.o 28 - 29 - obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o pelt.o 30 - obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o 31 - obj-$(CONFIG_SCHEDSTATS) += stats.o 32 - obj-$(CONFIG_SCHED_DEBUG) += debug.o 33 - obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o 34 - obj-$(CONFIG_CPU_FREQ) += cpufreq.o 35 - obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o 36 - obj-$(CONFIG_MEMBARRIER) += membarrier.o 37 - obj-$(CONFIG_CPU_ISOLATION) += isolation.o 38 - obj-$(CONFIG_PSI) += psi.o 39 - obj-$(CONFIG_SCHED_CORE) += core_sched.o 28 + # 29 + # Build efficiency: 30 + # 31 + # These compilation units have roughly the same size and complexity - so their 32 + # build parallelizes well and finishes roughly at once: 33 + # 34 + obj-y += core.o 35 + obj-y += fair.o 36 + obj-y += build_policy.o 37 + obj-y += build_utility.o
+1 -2
kernel/sched/autogroup.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 + 2 3 /* 3 4 * Auto-group scheduling implementation: 4 5 */ 5 - #include <linux/nospec.h> 6 - #include "sched.h" 7 6 8 7 unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1; 9 8 static struct autogroup autogroup_default;
+5
kernel/sched/autogroup.h
··· 1 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef _KERNEL_SCHED_AUTOGROUP_H 3 + #define _KERNEL_SCHED_AUTOGROUP_H 4 + 2 5 #ifdef CONFIG_SCHED_AUTOGROUP 3 6 4 7 struct autogroup { ··· 62 59 } 63 60 64 61 #endif /* CONFIG_SCHED_AUTOGROUP */ 62 + 63 + #endif /* _KERNEL_SCHED_AUTOGROUP_H */
+52
kernel/sched/build_policy.c
··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * These are the scheduling policy related scheduler files, built 4 + * in a single compilation unit for build efficiency reasons. 5 + * 6 + * ( Incidentally, the size of the compilation unit is roughly 7 + * comparable to core.c and fair.c, the other two big 8 + * compilation units. This helps balance build time, while 9 + * coalescing source files to amortize header inclusion 10 + * cost. ) 11 + * 12 + * core.c and fair.c are built separately. 13 + */ 14 + 15 + /* Headers: */ 16 + #include <linux/sched/clock.h> 17 + #include <linux/sched/cputime.h> 18 + #include <linux/sched/posix-timers.h> 19 + #include <linux/sched/rt.h> 20 + 21 + #include <linux/cpuidle.h> 22 + #include <linux/jiffies.h> 23 + #include <linux/livepatch.h> 24 + #include <linux/psi.h> 25 + #include <linux/seqlock_api.h> 26 + #include <linux/slab.h> 27 + #include <linux/suspend.h> 28 + #include <linux/tsacct_kern.h> 29 + #include <linux/vtime.h> 30 + 31 + #include <uapi/linux/sched/types.h> 32 + 33 + #include "sched.h" 34 + 35 + #include "autogroup.h" 36 + #include "stats.h" 37 + #include "pelt.h" 38 + 39 + /* Source code modules: */ 40 + 41 + #include "idle.c" 42 + 43 + #include "rt.c" 44 + 45 + #ifdef CONFIG_SMP 46 + # include "cpudeadline.c" 47 + # include "pelt.c" 48 + #endif 49 + 50 + #include "cputime.c" 51 + #include "deadline.c" 52 +
+109
kernel/sched/build_utility.c
··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * These are various utility functions of the scheduler, 4 + * built in a single compilation unit for build efficiency reasons. 5 + * 6 + * ( Incidentally, the size of the compilation unit is roughly 7 + * comparable to core.c, fair.c, smp.c and policy.c, the other 8 + * big compilation units. This helps balance build time, while 9 + * coalescing source files to amortize header inclusion 10 + * cost. ) 11 + */ 12 + #include <linux/sched/clock.h> 13 + #include <linux/sched/cputime.h> 14 + #include <linux/sched/debug.h> 15 + #include <linux/sched/isolation.h> 16 + #include <linux/sched/loadavg.h> 17 + #include <linux/sched/mm.h> 18 + #include <linux/sched/rseq_api.h> 19 + #include <linux/sched/task_stack.h> 20 + 21 + #include <linux/cpufreq.h> 22 + #include <linux/cpumask_api.h> 23 + #include <linux/cpuset.h> 24 + #include <linux/ctype.h> 25 + #include <linux/debugfs.h> 26 + #include <linux/energy_model.h> 27 + #include <linux/hashtable_api.h> 28 + #include <linux/irq.h> 29 + #include <linux/kobject_api.h> 30 + #include <linux/membarrier.h> 31 + #include <linux/mempolicy.h> 32 + #include <linux/nmi.h> 33 + #include <linux/nospec.h> 34 + #include <linux/proc_fs.h> 35 + #include <linux/psi.h> 36 + #include <linux/psi.h> 37 + #include <linux/ptrace_api.h> 38 + #include <linux/sched_clock.h> 39 + #include <linux/security.h> 40 + #include <linux/spinlock_api.h> 41 + #include <linux/swait_api.h> 42 + #include <linux/timex.h> 43 + #include <linux/utsname.h> 44 + #include <linux/wait_api.h> 45 + #include <linux/workqueue_api.h> 46 + 47 + #include <uapi/linux/prctl.h> 48 + #include <uapi/linux/sched/types.h> 49 + 50 + #include <asm/switch_to.h> 51 + 52 + #include "sched.h" 53 + #include "sched-pelt.h" 54 + #include "stats.h" 55 + #include "autogroup.h" 56 + 57 + #include "clock.c" 58 + 59 + #ifdef CONFIG_CGROUP_CPUACCT 60 + # include "cpuacct.c" 61 + #endif 62 + 63 + #ifdef CONFIG_CPU_FREQ 64 + # include "cpufreq.c" 65 + #endif 66 + 67 + #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL 68 + # include "cpufreq_schedutil.c" 69 + #endif 70 + 71 + #ifdef CONFIG_SCHED_DEBUG 72 + # include "debug.c" 73 + #endif 74 + 75 + #ifdef CONFIG_SCHEDSTATS 76 + # include "stats.c" 77 + #endif 78 + 79 + #include "loadavg.c" 80 + #include "completion.c" 81 + #include "swait.c" 82 + #include "wait_bit.c" 83 + #include "wait.c" 84 + 85 + #ifdef CONFIG_SMP 86 + # include "cpupri.c" 87 + # include "stop_task.c" 88 + # include "topology.c" 89 + #endif 90 + 91 + #ifdef CONFIG_SCHED_CORE 92 + # include "core_sched.c" 93 + #endif 94 + 95 + #ifdef CONFIG_PSI 96 + # include "psi.c" 97 + #endif 98 + 99 + #ifdef CONFIG_MEMBARRIER 100 + # include "membarrier.c" 101 + #endif 102 + 103 + #ifdef CONFIG_CPU_ISOLATION 104 + # include "isolation.c" 105 + #endif 106 + 107 + #ifdef CONFIG_SCHED_AUTOGROUP 108 + # include "autogroup.c" 109 + #endif
+21 -23
kernel/sched/clock.c
··· 53 53 * that is otherwise invisible (TSC gets stopped). 54 54 * 55 55 */ 56 - #include "sched.h" 57 - #include <linux/sched_clock.h> 58 56 59 57 /* 60 58 * Scheduler clock - returns current time in nanosec units. 61 59 * This is default implementation. 62 60 * Architectures and sub-architectures can override this. 63 61 */ 64 - unsigned long long __weak sched_clock(void) 62 + notrace unsigned long long __weak sched_clock(void) 65 63 { 66 64 return (unsigned long long)(jiffies - INITIAL_JIFFIES) 67 65 * (NSEC_PER_SEC / HZ); ··· 93 95 94 96 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 95 97 96 - static inline struct sched_clock_data *this_scd(void) 98 + notrace static inline struct sched_clock_data *this_scd(void) 97 99 { 98 100 return this_cpu_ptr(&sched_clock_data); 99 101 } 100 102 101 - static inline struct sched_clock_data *cpu_sdc(int cpu) 103 + notrace static inline struct sched_clock_data *cpu_sdc(int cpu) 102 104 { 103 105 return &per_cpu(sched_clock_data, cpu); 104 106 } 105 107 106 - int sched_clock_stable(void) 108 + notrace int sched_clock_stable(void) 107 109 { 108 110 return static_branch_likely(&__sched_clock_stable); 109 111 } 110 112 111 - static void __scd_stamp(struct sched_clock_data *scd) 113 + notrace static void __scd_stamp(struct sched_clock_data *scd) 112 114 { 113 115 scd->tick_gtod = ktime_get_ns(); 114 116 scd->tick_raw = sched_clock(); 115 117 } 116 118 117 - static void __set_sched_clock_stable(void) 119 + notrace static void __set_sched_clock_stable(void) 118 120 { 119 121 struct sched_clock_data *scd; 120 122 ··· 149 151 * The only way to fully avoid random clock jumps is to boot with: 150 152 * "tsc=unstable". 151 153 */ 152 - static void __sched_clock_work(struct work_struct *work) 154 + notrace static void __sched_clock_work(struct work_struct *work) 153 155 { 154 156 struct sched_clock_data *scd; 155 157 int cpu; ··· 175 177 176 178 static DECLARE_WORK(sched_clock_work, __sched_clock_work); 177 179 178 - static void __clear_sched_clock_stable(void) 180 + notrace static void __clear_sched_clock_stable(void) 179 181 { 180 182 if (!sched_clock_stable()) 181 183 return; ··· 184 186 schedule_work(&sched_clock_work); 185 187 } 186 188 187 - void clear_sched_clock_stable(void) 189 + notrace void clear_sched_clock_stable(void) 188 190 { 189 191 __sched_clock_stable_early = 0; 190 192 ··· 194 196 __clear_sched_clock_stable(); 195 197 } 196 198 197 - static void __sched_clock_gtod_offset(void) 199 + notrace static void __sched_clock_gtod_offset(void) 198 200 { 199 201 struct sched_clock_data *scd = this_scd(); 200 202 ··· 244 246 * min, max except they take wrapping into account 245 247 */ 246 248 247 - static inline u64 wrap_min(u64 x, u64 y) 249 + notrace static inline u64 wrap_min(u64 x, u64 y) 248 250 { 249 251 return (s64)(x - y) < 0 ? x : y; 250 252 } 251 253 252 - static inline u64 wrap_max(u64 x, u64 y) 254 + notrace static inline u64 wrap_max(u64 x, u64 y) 253 255 { 254 256 return (s64)(x - y) > 0 ? x : y; 255 257 } ··· 260 262 * - filter out backward motion 261 263 * - use the GTOD tick value to create a window to filter crazy TSC values 262 264 */ 263 - static u64 sched_clock_local(struct sched_clock_data *scd) 265 + notrace static u64 sched_clock_local(struct sched_clock_data *scd) 264 266 { 265 267 u64 now, clock, old_clock, min_clock, max_clock, gtod; 266 268 s64 delta; ··· 293 295 return clock; 294 296 } 295 297 296 - static u64 sched_clock_remote(struct sched_clock_data *scd) 298 + notrace static u64 sched_clock_remote(struct sched_clock_data *scd) 297 299 { 298 300 struct sched_clock_data *my_scd = this_scd(); 299 301 u64 this_clock, remote_clock; ··· 360 362 * 361 363 * See cpu_clock(). 362 364 */ 363 - u64 sched_clock_cpu(int cpu) 365 + notrace u64 sched_clock_cpu(int cpu) 364 366 { 365 367 struct sched_clock_data *scd; 366 368 u64 clock; ··· 384 386 } 385 387 EXPORT_SYMBOL_GPL(sched_clock_cpu); 386 388 387 - void sched_clock_tick(void) 389 + notrace void sched_clock_tick(void) 388 390 { 389 391 struct sched_clock_data *scd; 390 392 ··· 401 403 sched_clock_local(scd); 402 404 } 403 405 404 - void sched_clock_tick_stable(void) 406 + notrace void sched_clock_tick_stable(void) 405 407 { 406 408 if (!sched_clock_stable()) 407 409 return; ··· 421 423 /* 422 424 * We are going deep-idle (irqs are disabled): 423 425 */ 424 - void sched_clock_idle_sleep_event(void) 426 + notrace void sched_clock_idle_sleep_event(void) 425 427 { 426 428 sched_clock_cpu(smp_processor_id()); 427 429 } ··· 430 432 /* 431 433 * We just idled; resync with ktime. 432 434 */ 433 - void sched_clock_idle_wakeup_event(void) 435 + notrace void sched_clock_idle_wakeup_event(void) 434 436 { 435 437 unsigned long flags; 436 438 ··· 456 458 local_irq_enable(); 457 459 } 458 460 459 - u64 sched_clock_cpu(int cpu) 461 + notrace u64 sched_clock_cpu(int cpu) 460 462 { 461 463 if (!static_branch_likely(&sched_clock_running)) 462 464 return 0; ··· 474 476 * On bare metal this function should return the same as local_clock. 475 477 * Architectures and sub-architectures can override this. 476 478 */ 477 - u64 __weak running_clock(void) 479 + notrace u64 __weak running_clock(void) 478 480 { 479 481 return local_clock(); 480 482 }
+1 -1
kernel/sched/completion.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 + 2 3 /* 3 4 * Generic wait-for-completion handler; 4 5 * ··· 12 11 * typically be used for exclusion which gives rise to priority inversion. 13 12 * Waiting for completion is a typically sync point, but not an exclusion point. 14 13 */ 15 - #include "sched.h" 16 14 17 15 /** 18 16 * complete: - signals a single thread waiting on this completion
+71 -10
kernel/sched/core.c
··· 6 6 * 7 7 * Copyright (C) 1991-2002 Linus Torvalds 8 8 */ 9 - #define CREATE_TRACE_POINTS 10 - #include <trace/events/sched.h> 11 - #undef CREATE_TRACE_POINTS 9 + #include <linux/highmem.h> 10 + #include <linux/hrtimer_api.h> 11 + #include <linux/ktime_api.h> 12 + #include <linux/sched/signal.h> 13 + #include <linux/syscalls_api.h> 14 + #include <linux/debug_locks.h> 15 + #include <linux/prefetch.h> 16 + #include <linux/capability.h> 17 + #include <linux/pgtable_api.h> 18 + #include <linux/wait_bit.h> 19 + #include <linux/jiffies.h> 20 + #include <linux/spinlock_api.h> 21 + #include <linux/cpumask_api.h> 22 + #include <linux/lockdep_api.h> 23 + #include <linux/hardirq.h> 24 + #include <linux/softirq.h> 25 + #include <linux/refcount_api.h> 26 + #include <linux/topology.h> 27 + #include <linux/sched/clock.h> 28 + #include <linux/sched/cond_resched.h> 29 + #include <linux/sched/debug.h> 30 + #include <linux/sched/isolation.h> 31 + #include <linux/sched/loadavg.h> 32 + #include <linux/sched/mm.h> 33 + #include <linux/sched/nohz.h> 34 + #include <linux/sched/rseq_api.h> 35 + #include <linux/sched/rt.h> 12 36 13 - #include "sched.h" 14 - 15 - #include <linux/nospec.h> 16 37 #include <linux/blkdev.h> 17 - #include <linux/jump_label.h> 38 + #include <linux/context_tracking.h> 39 + #include <linux/cpuset.h> 40 + #include <linux/delayacct.h> 41 + #include <linux/init_task.h> 42 + #include <linux/interrupt.h> 43 + #include <linux/ioprio.h> 44 + #include <linux/kallsyms.h> 18 45 #include <linux/kcov.h> 46 + #include <linux/kprobes.h> 47 + #include <linux/llist_api.h> 48 + #include <linux/mmu_context.h> 49 + #include <linux/mmzone.h> 50 + #include <linux/mutex_api.h> 51 + #include <linux/nmi.h> 52 + #include <linux/nospec.h> 53 + #include <linux/perf_event_api.h> 54 + #include <linux/profile.h> 55 + #include <linux/psi.h> 56 + #include <linux/rcuwait_api.h> 57 + #include <linux/sched/wake_q.h> 19 58 #include <linux/scs.h> 59 + #include <linux/slab.h> 60 + #include <linux/syscalls.h> 61 + #include <linux/vtime.h> 62 + #include <linux/wait_api.h> 63 + #include <linux/workqueue_api.h> 64 + 65 + #ifdef CONFIG_PREEMPT_DYNAMIC 66 + # include <linux/entry-common.h> 67 + #endif 68 + 69 + #include <uapi/linux/sched/types.h> 20 70 21 71 #include <asm/switch_to.h> 22 72 #include <asm/tlb.h> 23 73 74 + #define CREATE_TRACE_POINTS 75 + #include <linux/sched/rseq_api.h> 76 + #include <trace/events/sched.h> 77 + #undef CREATE_TRACE_POINTS 78 + 79 + #include "sched.h" 80 + #include "stats.h" 81 + #include "autogroup.h" 82 + 83 + #include "autogroup.h" 84 + #include "pelt.h" 85 + #include "smp.h" 86 + #include "stats.h" 87 + 24 88 #include "../workqueue_internal.h" 25 89 #include "../../fs/io-wq.h" 26 90 #include "../smpboot.h" 27 - 28 - #include "pelt.h" 29 - #include "smp.h" 30 91 31 92 /* 32 93 * Export tracepoints that act as a bare tracehook (ie: have no trace event
-3
kernel/sched/core_sched.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 2 3 - #include <linux/prctl.h> 4 - #include "sched.h" 5 - 6 3 /* 7 4 * A simple wrapper around refcount. An allocated sched_core_cookie's 8 5 * address is used to compute the cookie of the task.
+1 -2
kernel/sched/cpuacct.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 + 2 3 /* 3 4 * CPU accounting code for task groups. 4 5 * 5 6 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh 6 7 * (balbir@in.ibm.com). 7 8 */ 8 - #include <asm/irq_regs.h> 9 - #include "sched.h" 10 9 11 10 /* Time spent by the tasks of the CPU accounting group executing in ... */ 12 11 enum cpuacct_stat_index {
+1 -2
kernel/sched/cpudeadline.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 2 /* 3 - * kernel/sched/cpudl.c 3 + * kernel/sched/cpudeadline.c 4 4 * 5 5 * Global CPU deadline management 6 6 * 7 7 * Author: Juri Lelli <j.lelli@sssup.it> 8 8 */ 9 - #include "sched.h" 10 9 11 10 static inline int parent(int i) 12 11 {
-3
kernel/sched/cpufreq.c
··· 5 5 * Copyright (C) 2016, Intel Corporation 6 6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> 7 7 */ 8 - #include <linux/cpufreq.h> 9 - 10 - #include "sched.h" 11 8 12 9 DEFINE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data); 13 10
-7
kernel/sched/cpufreq_schedutil.c
··· 6 6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> 7 7 */ 8 8 9 - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 - 11 - #include "sched.h" 12 - 13 - #include <linux/sched/cpufreq.h> 14 - #include <trace/events/power.h> 15 - 16 9 #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8) 17 10 18 11 struct sugov_tunables {
-1
kernel/sched/cpupri.c
··· 22 22 * worst case complexity of O(min(101, nr_domcpus)), though the scenario that 23 23 * yields the worst case search is fairly contrived. 24 24 */ 25 - #include "sched.h" 26 25 27 26 /* 28 27 * p->rt_priority p->prio newpri cpupri
-1
kernel/sched/cputime.c
··· 2 2 /* 3 3 * Simple CPU accounting cgroup controller 4 4 */ 5 - #include "sched.h" 6 5 7 6 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 8 7
-2
kernel/sched/deadline.c
··· 15 15 * Michael Trimarchi <michael@amarulasolutions.com>, 16 16 * Fabio Checconi <fchecconi@gmail.com> 17 17 */ 18 - #include "sched.h" 19 - #include "pelt.h" 20 18 21 19 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) 22 20 {
-1
kernel/sched/debug.c
··· 6 6 * 7 7 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar 8 8 */ 9 - #include "sched.h" 10 9 11 10 /* 12 11 * This allows printing both to /proc/sched_debug and
+31
kernel/sched/fair.c
··· 20 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 22 */ 23 + #include <linux/energy_model.h> 24 + #include <linux/mmap_lock.h> 25 + #include <linux/hugetlb_inline.h> 26 + #include <linux/jiffies.h> 27 + #include <linux/mm_api.h> 28 + #include <linux/highmem.h> 29 + #include <linux/spinlock_api.h> 30 + #include <linux/cpumask_api.h> 31 + #include <linux/lockdep_api.h> 32 + #include <linux/softirq.h> 33 + #include <linux/refcount_api.h> 34 + #include <linux/topology.h> 35 + #include <linux/sched/clock.h> 36 + #include <linux/sched/cond_resched.h> 37 + #include <linux/sched/cputime.h> 38 + #include <linux/sched/isolation.h> 39 + 40 + #include <linux/cpuidle.h> 41 + #include <linux/interrupt.h> 42 + #include <linux/mempolicy.h> 43 + #include <linux/mutex_api.h> 44 + #include <linux/profile.h> 45 + #include <linux/psi.h> 46 + #include <linux/ratelimit.h> 47 + 48 + #include <asm/switch_to.h> 49 + 50 + #include <linux/sched/cond_resched.h> 51 + 23 52 #include "sched.h" 53 + #include "stats.h" 54 + #include "autogroup.h" 24 55 25 56 /* 26 57 * Targeted preemption latency for CPU-bound tasks:
-3
kernel/sched/idle.c
··· 6 6 * (NOTE: these are not related to SCHED_IDLE batch scheduled 7 7 * tasks which are handled in sched/fair.c ) 8 8 */ 9 - #include "sched.h" 10 - 11 - #include <trace/events/power.h> 12 9 13 10 /* Linker adds these: start and end of __cpuidle functions */ 14 11 extern char __cpuidle_text_start[], __cpuidle_text_end[];
-1
kernel/sched/isolation.c
··· 7 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 8 * 9 9 */ 10 - #include "sched.h" 11 10 12 11 enum hk_flags { 13 12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
-1
kernel/sched/loadavg.c
··· 6 6 * figure. Its a silly number but people think its important. We go through 7 7 * great pains to make it work on big machines and tickless kernels. 8 8 */ 9 - #include "sched.h" 10 9 11 10 /* 12 11 * Global load-average calculations
-1
kernel/sched/membarrier.c
··· 4 4 * 5 5 * membarrier system call 6 6 */ 7 - #include "sched.h" 8 7 9 8 /* 10 9 * For documentation purposes, here are some membarrier ordering
-4
kernel/sched/pelt.c
··· 24 24 * Author: Vincent Guittot <vincent.guittot@linaro.org> 25 25 */ 26 26 27 - #include <linux/sched.h> 28 - #include "sched.h" 29 - #include "pelt.h" 30 - 31 27 /* 32 28 * Approximate: 33 29 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period)
-15
kernel/sched/psi.c
··· 137 137 * sampling of the aggregate task states would be. 138 138 */ 139 139 140 - #include "../workqueue_internal.h" 141 - #include <linux/sched/loadavg.h> 142 - #include <linux/seq_file.h> 143 - #include <linux/proc_fs.h> 144 - #include <linux/seqlock.h> 145 - #include <linux/uaccess.h> 146 - #include <linux/cgroup.h> 147 - #include <linux/module.h> 148 - #include <linux/sched.h> 149 - #include <linux/ctype.h> 150 - #include <linux/file.h> 151 - #include <linux/poll.h> 152 - #include <linux/psi.h> 153 - #include "sched.h" 154 - 155 140 static int psi_bug __read_mostly; 156 141 157 142 DEFINE_STATIC_KEY_FALSE(psi_disabled);
-3
kernel/sched/rt.c
··· 3 3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR 4 4 * policies) 5 5 */ 6 - #include "sched.h" 7 - 8 - #include "pelt.h" 9 6 10 7 int sched_rr_timeslice = RR_TIMESLICE; 11 8 int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
+61 -53
kernel/sched/sched.h
··· 2 2 /* 3 3 * Scheduler internal types and methods: 4 4 */ 5 - #include <linux/sched.h> 5 + #ifndef _KERNEL_SCHED_SCHED_H 6 + #define _KERNEL_SCHED_SCHED_H 6 7 8 + #include <linux/sched/affinity.h> 7 9 #include <linux/sched/autogroup.h> 8 - #include <linux/sched/clock.h> 9 - #include <linux/sched/coredump.h> 10 10 #include <linux/sched/cpufreq.h> 11 - #include <linux/sched/cputime.h> 12 11 #include <linux/sched/deadline.h> 13 - #include <linux/sched/debug.h> 14 - #include <linux/sched/hotplug.h> 15 - #include <linux/sched/idle.h> 16 - #include <linux/sched/init.h> 17 - #include <linux/sched/isolation.h> 18 - #include <linux/sched/jobctl.h> 12 + #include <linux/sched.h> 19 13 #include <linux/sched/loadavg.h> 20 14 #include <linux/sched/mm.h> 21 - #include <linux/sched/nohz.h> 22 - #include <linux/sched/numa_balancing.h> 23 - #include <linux/sched/prio.h> 24 - #include <linux/sched/rt.h> 15 + #include <linux/sched/rseq_api.h> 25 16 #include <linux/sched/signal.h> 26 17 #include <linux/sched/smt.h> 27 18 #include <linux/sched/stat.h> 28 19 #include <linux/sched/sysctl.h> 20 + #include <linux/sched/task_flags.h> 29 21 #include <linux/sched/task.h> 30 - #include <linux/sched/task_stack.h> 31 22 #include <linux/sched/topology.h> 32 - #include <linux/sched/user.h> 33 - #include <linux/sched/wake_q.h> 34 - #include <linux/sched/xacct.h> 35 23 36 - #include <uapi/linux/sched/types.h> 37 - 38 - #include <linux/binfmts.h> 39 - #include <linux/bitops.h> 40 - #include <linux/compat.h> 41 - #include <linux/context_tracking.h> 24 + #include <linux/atomic.h> 25 + #include <linux/bitmap.h> 26 + #include <linux/bug.h> 27 + #include <linux/capability.h> 28 + #include <linux/cgroup_api.h> 29 + #include <linux/cgroup.h> 42 30 #include <linux/cpufreq.h> 43 - #include <linux/cpuidle.h> 44 - #include <linux/cpuset.h> 31 + #include <linux/cpumask_api.h> 45 32 #include <linux/ctype.h> 46 - #include <linux/debugfs.h> 47 - #include <linux/delayacct.h> 48 - #include <linux/energy_model.h> 49 - #include <linux/init_task.h> 50 - #include <linux/kprobes.h> 33 + #include <linux/file.h> 34 + #include <linux/fs_api.h> 35 + #include <linux/hrtimer_api.h> 36 + #include <linux/interrupt.h> 37 + #include <linux/irq_work.h> 38 + #include <linux/jiffies.h> 39 + #include <linux/kref_api.h> 51 40 #include <linux/kthread.h> 52 - #include <linux/membarrier.h> 53 - #include <linux/migrate.h> 54 - #include <linux/mmu_context.h> 55 - #include <linux/nmi.h> 41 + #include <linux/ktime_api.h> 42 + #include <linux/lockdep_api.h> 43 + #include <linux/lockdep.h> 44 + #include <linux/minmax.h> 45 + #include <linux/mm.h> 46 + #include <linux/module.h> 47 + #include <linux/mutex_api.h> 48 + #include <linux/plist.h> 49 + #include <linux/poll.h> 56 50 #include <linux/proc_fs.h> 57 - #include <linux/prefetch.h> 58 51 #include <linux/profile.h> 59 52 #include <linux/psi.h> 60 - #include <linux/ratelimit.h> 61 - #include <linux/rcupdate_wait.h> 62 - #include <linux/security.h> 53 + #include <linux/rcupdate.h> 54 + #include <linux/seq_file.h> 55 + #include <linux/seqlock.h> 56 + #include <linux/softirq.h> 57 + #include <linux/spinlock_api.h> 58 + #include <linux/static_key.h> 63 59 #include <linux/stop_machine.h> 64 - #include <linux/suspend.h> 65 - #include <linux/swait.h> 60 + #include <linux/syscalls_api.h> 66 61 #include <linux/syscalls.h> 67 - #include <linux/task_work.h> 68 - #include <linux/tsacct_kern.h> 62 + #include <linux/tick.h> 63 + #include <linux/topology.h> 64 + #include <linux/types.h> 65 + #include <linux/u64_stats_sync_api.h> 66 + #include <linux/uaccess.h> 67 + #include <linux/wait_api.h> 68 + #include <linux/wait_bit.h> 69 + #include <linux/workqueue_api.h> 69 70 70 - #include <asm/tlb.h> 71 + #include <trace/events/power.h> 72 + #include <trace/events/sched.h> 73 + 74 + #include "../workqueue_internal.h" 75 + 76 + #ifdef CONFIG_CGROUP_SCHED 77 + #include <linux/cgroup.h> 78 + #include <linux/psi.h> 79 + #endif 80 + 81 + #ifdef CONFIG_SCHED_DEBUG 82 + # include <linux/static_key.h> 83 + #endif 71 84 72 85 #ifdef CONFIG_PARAVIRT 73 86 # include <asm/paravirt.h> 87 + # include <asm/paravirt_api_clock.h> 74 88 #endif 75 89 76 90 #include "cpupri.h" 77 91 #include "cpudeadline.h" 78 92 79 - #include <trace/events/sched.h> 80 - 81 93 #ifdef CONFIG_SCHED_DEBUG 82 - # define SCHED_WARN_ON(x) WARN_ONCE(x, #x) 94 + # define SCHED_WARN_ON(x) WARN_ONCE(x, #x) 83 95 #else 84 - # define SCHED_WARN_ON(x) ({ (void)(x), 0; }) 96 + # define SCHED_WARN_ON(x) ({ (void)(x), 0; }) 85 97 #endif 86 98 87 99 struct rq; ··· 340 328 extern int dl_cpu_busy(int cpu, struct task_struct *p); 341 329 342 330 #ifdef CONFIG_CGROUP_SCHED 343 - 344 - #include <linux/cgroup.h> 345 - #include <linux/psi.h> 346 331 347 332 struct cfs_rq; 348 333 struct rt_rq; ··· 1841 1832 #endif 1842 1833 1843 1834 #include "stats.h" 1844 - #include "autogroup.h" 1845 1835 1846 1836 #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) 1847 1837 ··· 1936 1928 * Tunables that become constants when CONFIG_SCHED_DEBUG is off: 1937 1929 */ 1938 1930 #ifdef CONFIG_SCHED_DEBUG 1939 - # include <linux/static_key.h> 1940 1931 # define const_debug __read_mostly 1941 1932 #else 1942 1933 # define const_debug const ··· 3093 3086 extern void sched_dynamic_update(int mode); 3094 3087 #endif 3095 3088 3089 + #endif /* _KERNEL_SCHED_SCHED_H */
-1
kernel/sched/stats.c
··· 2 2 /* 3 3 * /proc/schedstat implementation 4 4 */ 5 - #include "sched.h" 6 5 7 6 void __update_stats_wait_start(struct rq *rq, struct task_struct *p, 8 7 struct sched_statistics *stats)
+4
kernel/sched/stats.h
··· 1 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef _KERNEL_STATS_H 3 + #define _KERNEL_STATS_H 2 4 3 5 #ifdef CONFIG_SCHEDSTATS 4 6 ··· 300 298 # define sched_info_dequeue(rq, t) do { } while (0) 301 299 # define sched_info_switch(rq, t, next) do { } while (0) 302 300 #endif /* CONFIG_SCHED_INFO */ 301 + 302 + #endif /* _KERNEL_STATS_H */
-1
kernel/sched/stop_task.c
··· 7 7 * 8 8 * See kernel/stop_machine.c 9 9 */ 10 - #include "sched.h" 11 10 12 11 #ifdef CONFIG_SMP 13 12 static int
-1
kernel/sched/swait.c
··· 2 2 /* 3 3 * <linux/swait.h> (simple wait queues ) implementation: 4 4 */ 5 - #include "sched.h" 6 5 7 6 void __init_swait_queue_head(struct swait_queue_head *q, const char *name, 8 7 struct lock_class_key *key)
-1
kernel/sched/topology.c
··· 2 2 /* 3 3 * Scheduler topology setup/handling methods 4 4 */ 5 - #include "sched.h" 6 5 7 6 DEFINE_MUTEX(sched_domains_mutex); 8 7
-1
kernel/sched/wait.c
··· 4 4 * 5 5 * (C) 2004 Nadia Yvette Chambers, Oracle 6 6 */ 7 - #include "sched.h" 8 7 9 8 void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) 10 9 {
+1 -1
kernel/sched/wait_bit.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 + 2 3 /* 3 4 * The implementation of the wait_bit*() and related waiting APIs: 4 5 */ 5 - #include "sched.h" 6 6 7 7 #define WAIT_TABLE_BITS 8 8 8 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)