tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * sched_clock.h: support for extending counters to full 64-bit ns counter
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8#ifndef ASM_SCHED_CLOCK
9#define ASM_SCHED_CLOCK
10
11#include <linux/kernel.h>
12#include <linux/types.h>
13
14struct clock_data {
15 u64 epoch_ns;
16 u32 epoch_cyc;
17 u32 epoch_cyc_copy;
18 u32 mult;
19 u32 shift;
20};
21
22#define DEFINE_CLOCK_DATA(name) struct clock_data name
23
24static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
25{
26 return (cyc * mult) >> shift;
27}
28
29/*
30 * Atomically update the sched_clock epoch. Your update callback will
31 * be called from a timer before the counter wraps - read the current
32 * counter value, and call this function to safely move the epochs
33 * forward. Only use this from the update callback.
34 */
35static inline void update_sched_clock(struct clock_data *cd, u32 cyc, u32 mask)
36{
37 unsigned long flags;
38 u64 ns = cd->epoch_ns +
39 cyc_to_ns((cyc - cd->epoch_cyc) & mask, cd->mult, cd->shift);
40
41 /*
42 * Write epoch_cyc and epoch_ns in a way that the update is
43 * detectable in cyc_to_fixed_sched_clock().
44 */
45 raw_local_irq_save(flags);
46 cd->epoch_cyc = cyc;
47 smp_wmb();
48 cd->epoch_ns = ns;
49 smp_wmb();
50 cd->epoch_cyc_copy = cyc;
51 raw_local_irq_restore(flags);
52}
53
54/*
55 * If your clock rate is known at compile time, using this will allow
56 * you to optimize the mult/shift loads away. This is paired with
57 * init_fixed_sched_clock() to ensure that your mult/shift are correct.
58 */
59static inline unsigned long long cyc_to_fixed_sched_clock(struct clock_data *cd,
60 u32 cyc, u32 mask, u32 mult, u32 shift)
61{
62 u64 epoch_ns;
63 u32 epoch_cyc;
64
65 /*
66 * Load the epoch_cyc and epoch_ns atomically. We do this by
67 * ensuring that we always write epoch_cyc, epoch_ns and
68 * epoch_cyc_copy in strict order, and read them in strict order.
69 * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
70 * the middle of an update, and we should repeat the load.
71 */
72 do {
73 epoch_cyc = cd->epoch_cyc;
74 smp_rmb();
75 epoch_ns = cd->epoch_ns;
76 smp_rmb();
77 } while (epoch_cyc != cd->epoch_cyc_copy);
78
79 return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, mult, shift);
80}
81
82/*
83 * Otherwise, you need to use this, which will obtain the mult/shift
84 * from the clock_data structure. Use init_sched_clock() with this.
85 */
86static inline unsigned long long cyc_to_sched_clock(struct clock_data *cd,
87 u32 cyc, u32 mask)
88{
89 return cyc_to_fixed_sched_clock(cd, cyc, mask, cd->mult, cd->shift);
90}
91
92/*
93 * Initialize the clock data - calculate the appropriate multiplier
94 * and shift. Also setup a timer to ensure that the epoch is refreshed
95 * at the appropriate time interval, which will call your update
96 * handler.
97 */
98void init_sched_clock(struct clock_data *, void (*)(void),
99 unsigned int, unsigned long);
100
101/*
102 * Use this initialization function rather than init_sched_clock() if
103 * you're using cyc_to_fixed_sched_clock, which will warn if your
104 * constants are incorrect.
105 */
106static inline void init_fixed_sched_clock(struct clock_data *cd,
107 void (*update)(void), unsigned int bits, unsigned long rate,
108 u32 mult, u32 shift)
109{
110 init_sched_clock(cd, update, bits, rate);
111 if (cd->mult != mult || cd->shift != shift) {
112 pr_crit("sched_clock: wrong multiply/shift: %u>>%u vs calculated %u>>%u\n"
113 "sched_clock: fix multiply/shift to avoid scheduler hiccups\n",
114 mult, shift, cd->mult, cd->shift);
115 }
116}
117
118extern void sched_clock_postinit(void);
119
120#endif