tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __VDSO_HELPERS_H
3#define __VDSO_HELPERS_H
4
5#ifndef __ASSEMBLY__
6
7#include <asm/barrier.h>
8#include <vdso/datapage.h>
9
10static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
11{
12 u32 seq;
13
14 while (unlikely((seq = READ_ONCE(vc->seq)) & 1))
15 cpu_relax();
16
17 smp_rmb();
18 return seq;
19}
20
21static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
22 u32 start)
23{
24 u32 seq;
25
26 smp_rmb();
27 seq = READ_ONCE(vc->seq);
28 return seq != start;
29}
30
31static __always_inline void vdso_write_begin(struct vdso_time_data *vd)
32{
33 struct vdso_clock *vc = vd->clock_data;
34
35 /*
36 * WRITE_ONCE() is required otherwise the compiler can validly tear
37 * updates to vd[x].seq and it is possible that the value seen by the
38 * reader is inconsistent.
39 */
40 WRITE_ONCE(vc[CS_HRES_COARSE].seq, vc[CS_HRES_COARSE].seq + 1);
41 WRITE_ONCE(vc[CS_RAW].seq, vc[CS_RAW].seq + 1);
42 smp_wmb();
43}
44
45static __always_inline void vdso_write_end(struct vdso_time_data *vd)
46{
47 struct vdso_clock *vc = vd->clock_data;
48
49 smp_wmb();
50 /*
51 * WRITE_ONCE() is required otherwise the compiler can validly tear
52 * updates to vd[x].seq and it is possible that the value seen by the
53 * reader is inconsistent.
54 */
55 WRITE_ONCE(vc[CS_HRES_COARSE].seq, vc[CS_HRES_COARSE].seq + 1);
56 WRITE_ONCE(vc[CS_RAW].seq, vc[CS_RAW].seq + 1);
57}
58
59#endif /* !__ASSEMBLY__ */
60
61#endif /* __VDSO_HELPERS_H */