tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _TOOLS_LINUX_ASM_AARCH64_BARRIER_H
3#define _TOOLS_LINUX_ASM_AARCH64_BARRIER_H
4
5/*
6 * From tools/perf/perf-sys.h, last modified in:
7 * f428ebd184c82a7914b2aa7e9f868918aaf7ea78 perf tools: Fix AAAAARGH64 memory barriers
8 *
9 * XXX: arch/arm64/include/asm/barrier.h in the kernel sources use dsb, is this
10 * a case like for arm32 where we do things differently in userspace?
11 */
12
13#define mb() asm volatile("dmb ish" ::: "memory")
14#define wmb() asm volatile("dmb ishst" ::: "memory")
15#define rmb() asm volatile("dmb ishld" ::: "memory")
16
17/*
18 * Kernel uses dmb variants on arm64 for smp_*() barriers. Pretty much the same
19 * implementation as above mb()/wmb()/rmb(), though for the latter kernel uses
20 * dsb. In any case, should above mb()/wmb()/rmb() change, make sure the below
21 * smp_*() don't.
22 */
23#define smp_mb() asm volatile("dmb ish" ::: "memory")
24#define smp_wmb() asm volatile("dmb ishst" ::: "memory")
25#define smp_rmb() asm volatile("dmb ishld" ::: "memory")
26
27#define smp_store_release(p, v) \
28do { \
29 union { typeof(*p) __val; char __c[1]; } __u = \
30 { .__val = (v) }; \
31 \
32 switch (sizeof(*p)) { \
33 case 1: \
34 asm volatile ("stlrb %w1, %0" \
35 : "=Q" (*p) \
36 : "r" (*(__u8_alias_t *)__u.__c) \
37 : "memory"); \
38 break; \
39 case 2: \
40 asm volatile ("stlrh %w1, %0" \
41 : "=Q" (*p) \
42 : "r" (*(__u16_alias_t *)__u.__c) \
43 : "memory"); \
44 break; \
45 case 4: \
46 asm volatile ("stlr %w1, %0" \
47 : "=Q" (*p) \
48 : "r" (*(__u32_alias_t *)__u.__c) \
49 : "memory"); \
50 break; \
51 case 8: \
52 asm volatile ("stlr %1, %0" \
53 : "=Q" (*p) \
54 : "r" (*(__u64_alias_t *)__u.__c) \
55 : "memory"); \
56 break; \
57 default: \
58 /* Only to shut up gcc ... */ \
59 mb(); \
60 break; \
61 } \
62} while (0)
63
64#define smp_load_acquire(p) \
65({ \
66 union { typeof(*p) __val; char __c[1]; } __u = \
67 { .__c = { 0 } }; \
68 \
69 switch (sizeof(*p)) { \
70 case 1: \
71 asm volatile ("ldarb %w0, %1" \
72 : "=r" (*(__u8_alias_t *)__u.__c) \
73 : "Q" (*p) : "memory"); \
74 break; \
75 case 2: \
76 asm volatile ("ldarh %w0, %1" \
77 : "=r" (*(__u16_alias_t *)__u.__c) \
78 : "Q" (*p) : "memory"); \
79 break; \
80 case 4: \
81 asm volatile ("ldar %w0, %1" \
82 : "=r" (*(__u32_alias_t *)__u.__c) \
83 : "Q" (*p) : "memory"); \
84 break; \
85 case 8: \
86 asm volatile ("ldar %0, %1" \
87 : "=r" (*(__u64_alias_t *)__u.__c) \
88 : "Q" (*p) : "memory"); \
89 break; \
90 default: \
91 /* Only to shut up gcc ... */ \
92 mb(); \
93 break; \
94 } \
95 __u.__val; \
96})
97
98#endif /* _TOOLS_LINUX_ASM_AARCH64_BARRIER_H */