tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
x86/lib/memset_64.S: Convert to ALTERNATIVE_2 macro
Make alternatives replace single JMPs instead of whole memset functions,
thus decreasing the amount of instructions copied during patching time
at boot.
While at it, make it use the REP_GOOD version by default which means
alternatives NOP out the JMP to the other versions, as REP_GOOD is set
by default on the majority of relevant x86 processors.
Signed-off-by: Borislav Petkov <bp@suse.de>
+24
-37
+24
-37
arch/x86/lib/memset_64.S
+24
-37
arch/x86/lib/memset_64.S
···
5
5
#include <asm/cpufeature.h>
6
6
#include <asm/alternative-asm.h>
7
7
8
+
.weak memset
9
+
8
10
/*
9
11
* ISO C memset - set a memory block to a byte value. This function uses fast
10
12
* string to get better performance than the original function. The code is
11
13
* simpler and shorter than the orignal function as well.
12
-
*
14
+
*
13
15
* rdi destination
14
-
* rsi value (char)
15
-
* rdx count (bytes)
16
-
*
16
+
* rsi value (char)
17
+
* rdx count (bytes)
18
+
*
17
19
* rax original destination
18
-
*/
19
-
.section .altinstr_replacement, "ax", @progbits
20
-
.Lmemset_c:
20
+
*/
21
+
ENTRY(memset)
22
+
ENTRY(__memset)
23
+
/*
24
+
* Some CPUs support enhanced REP MOVSB/STOSB feature. It is recommended
25
+
* to use it when possible. If not available, use fast string instructions.
26
+
*
27
+
* Otherwise, use original memset function.
28
+
*/
29
+
ALTERNATIVE_2 "jmp memset_orig", "", X86_FEATURE_REP_GOOD, \
30
+
"jmp memset_erms", X86_FEATURE_ERMS
31
+
21
32
movq %rdi,%r9
22
33
movq %rdx,%rcx
23
34
andl $7,%edx
···
42
31
rep stosb
43
32
movq %r9,%rax
44
33
ret
45
-
.Lmemset_e:
46
-
.previous
34
+
ENDPROC(memset)
35
+
ENDPROC(__memset)
47
36
48
37
/*
49
38
* ISO C memset - set a memory block to a byte value. This function uses
···
56
45
*
57
46
* rax original destination
58
47
*/
59
-
.section .altinstr_replacement, "ax", @progbits
60
-
.Lmemset_c_e:
48
+
ENTRY(memset_erms)
61
49
movq %rdi,%r9
62
50
movb %sil,%al
63
51
movq %rdx,%rcx
64
52
rep stosb
65
53
movq %r9,%rax
66
54
ret
67
-
.Lmemset_e_e:
68
-
.previous
55
+
ENDPROC(memset_erms)
69
56
70
-
.weak memset
71
-
72
-
ENTRY(memset)
73
-
ENTRY(__memset)
57
+
ENTRY(memset_orig)
74
58
CFI_STARTPROC
75
59
movq %rdi,%r10
76
60
···
140
134
jmp .Lafter_bad_alignment
141
135
.Lfinal:
142
136
CFI_ENDPROC
143
-
ENDPROC(memset)
144
-
ENDPROC(__memset)
145
-
146
-
/* Some CPUs support enhanced REP MOVSB/STOSB feature.
147
-
* It is recommended to use this when possible.
148
-
*
149
-
* If enhanced REP MOVSB/STOSB feature is not available, use fast string
150
-
* instructions.
151
-
*
152
-
* Otherwise, use original memset function.
153
-
*
154
-
* In .altinstructions section, ERMS feature is placed after REG_GOOD
155
-
* feature to implement the right patch order.
156
-
*/
157
-
.section .altinstructions,"a"
158
-
altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
159
-
.Lfinal-__memset,.Lmemset_e-.Lmemset_c,0
160
-
altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
161
-
.Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e,0
162
-
.previous
137
+
ENDPROC(memset_orig)