Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
4/*
5 * Jump label support
6 *
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 *
10 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support the result
12 * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13 * defaults to false - and the true block is placed out of line).
14 *
15 * However at runtime we can change the branch target using
16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17 * object and for as long as there are references all branches referring to
18 * that particular key will point to the (out of line) true block.
19 *
20 * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total
24 * effect is a single NOP of appropriate size. The on case will patch in a jump
25 * to the out-of-line block.
26 *
27 * When the control is directly exposed to userspace it is prudent to delay the
28 * decrement to avoid high frequency code modifications which can (and do)
29 * cause significant performance degradation. Struct static_key_deferred and
30 * static_key_slow_dec_deferred() provide for this.
31 *
32 * Lacking toolchain and or architecture support, it falls back to a simple
33 * conditional branch.
34 *
35 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 *
37 * if (static_key_true(&my_key)) {
38 * }
39 *
40 * will result in the true case being in-line and starts the key with a single
41 * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 * allowed.
43 *
44 * Not initializing the key (static data is initialized to 0s anyway) is the
45 * same as using STATIC_KEY_INIT_FALSE.
46 *
47*/
48
49#include <linux/types.h>
50#include <linux/compiler.h>
51#include <linux/bug.h>
52
53extern bool static_key_initialized;
54
55#define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
56 "%s used before call to jump_label_init", \
57 __func__)
58
59#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
60
61struct static_key {
62 atomic_t enabled;
63/* Set lsb bit to 1 if branch is default true, 0 ot */
64 struct jump_entry *entries;
65#ifdef CONFIG_MODULES
66 struct static_key_mod *next;
67#endif
68};
69
70# include <asm/jump_label.h>
71# define HAVE_JUMP_LABEL
72#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
73
74enum jump_label_type {
75 JUMP_LABEL_DISABLE = 0,
76 JUMP_LABEL_ENABLE,
77};
78
79struct module;
80
81#include <linux/atomic.h>
82#ifdef HAVE_JUMP_LABEL
83
84#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL
85#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL
86#define JUMP_LABEL_TYPE_MASK 1UL
87
88static
89inline struct jump_entry *jump_label_get_entries(struct static_key *key)
90{
91 return (struct jump_entry *)((unsigned long)key->entries
92 & ~JUMP_LABEL_TYPE_MASK);
93}
94
95static inline bool jump_label_get_branch_default(struct static_key *key)
96{
97 if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
98 JUMP_LABEL_TYPE_TRUE_BRANCH)
99 return true;
100 return false;
101}
102
103static __always_inline bool static_key_false(struct static_key *key)
104{
105 return arch_static_branch(key);
106}
107
108static __always_inline bool static_key_true(struct static_key *key)
109{
110 return !static_key_false(key);
111}
112
113extern struct jump_entry __start___jump_table[];
114extern struct jump_entry __stop___jump_table[];
115
116extern void jump_label_init(void);
117extern void jump_label_lock(void);
118extern void jump_label_unlock(void);
119extern void arch_jump_label_transform(struct jump_entry *entry,
120 enum jump_label_type type);
121extern void arch_jump_label_transform_static(struct jump_entry *entry,
122 enum jump_label_type type);
123extern int jump_label_text_reserved(void *start, void *end);
124extern void static_key_slow_inc(struct static_key *key);
125extern void static_key_slow_dec(struct static_key *key);
126extern void jump_label_apply_nops(struct module *mod);
127
128#define STATIC_KEY_INIT_TRUE ((struct static_key) \
129 { .enabled = ATOMIC_INIT(1), \
130 .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
131#define STATIC_KEY_INIT_FALSE ((struct static_key) \
132 { .enabled = ATOMIC_INIT(0), \
133 .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
134
135#else /* !HAVE_JUMP_LABEL */
136
137struct static_key {
138 atomic_t enabled;
139};
140
141static __always_inline void jump_label_init(void)
142{
143 static_key_initialized = true;
144}
145
146static __always_inline bool static_key_false(struct static_key *key)
147{
148 if (unlikely(atomic_read(&key->enabled) > 0))
149 return true;
150 return false;
151}
152
153static __always_inline bool static_key_true(struct static_key *key)
154{
155 if (likely(atomic_read(&key->enabled) > 0))
156 return true;
157 return false;
158}
159
160static inline void static_key_slow_inc(struct static_key *key)
161{
162 STATIC_KEY_CHECK_USE();
163 atomic_inc(&key->enabled);
164}
165
166static inline void static_key_slow_dec(struct static_key *key)
167{
168 STATIC_KEY_CHECK_USE();
169 atomic_dec(&key->enabled);
170}
171
172static inline int jump_label_text_reserved(void *start, void *end)
173{
174 return 0;
175}
176
177static inline void jump_label_lock(void) {}
178static inline void jump_label_unlock(void) {}
179
180static inline int jump_label_apply_nops(struct module *mod)
181{
182 return 0;
183}
184
185#define STATIC_KEY_INIT_TRUE ((struct static_key) \
186 { .enabled = ATOMIC_INIT(1) })
187#define STATIC_KEY_INIT_FALSE ((struct static_key) \
188 { .enabled = ATOMIC_INIT(0) })
189
190#endif /* HAVE_JUMP_LABEL */
191
192#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
193#define jump_label_enabled static_key_enabled
194
195static inline bool static_key_enabled(struct static_key *key)
196{
197 return (atomic_read(&key->enabled) > 0);
198}
199
200#endif /* _LINUX_JUMP_LABEL_H */