Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
4/*
5 * Jump label support
6 *
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 *
10 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support the result
12 * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13 * defaults to false - and the true block is placed out of line).
14 *
15 * However at runtime we can change the branch target using
16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17 * object and for as long as there are references all branches referring to
18 * that particular key will point to the (out of line) true block.
19 *
20 * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total
24 * effect is a single NOP of appropriate size. The on case will patch in a jump
25 * to the out-of-line block.
26 *
27 * When the control is directly exposed to userspace it is prudent to delay the
28 * decrement to avoid high frequency code modifications which can (and do)
29 * cause significant performance degradation. Struct static_key_deferred and
30 * static_key_slow_dec_deferred() provide for this.
31 *
32 * Lacking toolchain and or architecture support, it falls back to a simple
33 * conditional branch.
34 *
35 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 *
37 * if (static_key_true(&my_key)) {
38 * }
39 *
40 * will result in the true case being in-line and starts the key with a single
41 * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 * allowed.
43 *
44 * Not initializing the key (static data is initialized to 0s anyway) is the
45 * same as using STATIC_KEY_INIT_FALSE.
46 *
47*/
48
49#include <linux/types.h>
50#include <linux/compiler.h>
51
52#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
53
54struct static_key {
55 atomic_t enabled;
56/* Set lsb bit to 1 if branch is default true, 0 ot */
57 struct jump_entry *entries;
58#ifdef CONFIG_MODULES
59 struct static_key_mod *next;
60#endif
61};
62
63# include <asm/jump_label.h>
64# define HAVE_JUMP_LABEL
65#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
66
67enum jump_label_type {
68 JUMP_LABEL_DISABLE = 0,
69 JUMP_LABEL_ENABLE,
70};
71
72struct module;
73
74#include <linux/atomic.h>
75#ifdef HAVE_JUMP_LABEL
76
77#define JUMP_LABEL_TRUE_BRANCH 1UL
78
79static
80inline struct jump_entry *jump_label_get_entries(struct static_key *key)
81{
82 return (struct jump_entry *)((unsigned long)key->entries
83 & ~JUMP_LABEL_TRUE_BRANCH);
84}
85
86static inline bool jump_label_get_branch_default(struct static_key *key)
87{
88 if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
89 return true;
90 return false;
91}
92
93static __always_inline bool static_key_false(struct static_key *key)
94{
95 return arch_static_branch(key);
96}
97
98static __always_inline bool static_key_true(struct static_key *key)
99{
100 return !static_key_false(key);
101}
102
103extern struct jump_entry __start___jump_table[];
104extern struct jump_entry __stop___jump_table[];
105
106extern void jump_label_init(void);
107extern void jump_label_lock(void);
108extern void jump_label_unlock(void);
109extern void arch_jump_label_transform(struct jump_entry *entry,
110 enum jump_label_type type);
111extern void arch_jump_label_transform_static(struct jump_entry *entry,
112 enum jump_label_type type);
113extern int jump_label_text_reserved(void *start, void *end);
114extern void static_key_slow_inc(struct static_key *key);
115extern void static_key_slow_dec(struct static_key *key);
116extern void jump_label_apply_nops(struct module *mod);
117
118#define STATIC_KEY_INIT_TRUE ((struct static_key) \
119 { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
120#define STATIC_KEY_INIT_FALSE ((struct static_key) \
121 { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
122
123#else /* !HAVE_JUMP_LABEL */
124
125struct static_key {
126 atomic_t enabled;
127};
128
129static __always_inline void jump_label_init(void)
130{
131}
132
133static __always_inline bool static_key_false(struct static_key *key)
134{
135 if (unlikely(atomic_read(&key->enabled)) > 0)
136 return true;
137 return false;
138}
139
140static __always_inline bool static_key_true(struct static_key *key)
141{
142 if (likely(atomic_read(&key->enabled)) > 0)
143 return true;
144 return false;
145}
146
147static inline void static_key_slow_inc(struct static_key *key)
148{
149 atomic_inc(&key->enabled);
150}
151
152static inline void static_key_slow_dec(struct static_key *key)
153{
154 atomic_dec(&key->enabled);
155}
156
157static inline int jump_label_text_reserved(void *start, void *end)
158{
159 return 0;
160}
161
162static inline void jump_label_lock(void) {}
163static inline void jump_label_unlock(void) {}
164
165static inline int jump_label_apply_nops(struct module *mod)
166{
167 return 0;
168}
169
170#define STATIC_KEY_INIT_TRUE ((struct static_key) \
171 { .enabled = ATOMIC_INIT(1) })
172#define STATIC_KEY_INIT_FALSE ((struct static_key) \
173 { .enabled = ATOMIC_INIT(0) })
174
175#endif /* HAVE_JUMP_LABEL */
176
177#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
178#define jump_label_enabled static_key_enabled
179
180static inline bool static_key_enabled(struct static_key *key)
181{
182 return (atomic_read(&key->enabled) > 0);
183}
184
185#endif /* _LINUX_JUMP_LABEL_H */