tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
fork
Configure Feed
Select the types of activity you want to include in your feed.
1/*
2 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
3 * Copyright 2002 Andi Kleen, SuSE Labs.
4 * Subject to the GNU Public License v2.
5 *
6 * Functions to copy from and to user space.
7 */
8
9#include <linux/linkage.h>
10#include <asm/current.h>
11#include <asm/asm-offsets.h>
12#include <asm/thread_info.h>
13#include <asm/cpufeature.h>
14#include <asm/alternative-asm.h>
15#include <asm/asm.h>
16#include <asm/smap.h>
17
18/* Standard copy_to_user with segment limit checking */
19ENTRY(_copy_to_user)
20 GET_THREAD_INFO(%rax)
21 movq %rdi,%rcx
22 addq %rdx,%rcx
23 jc bad_to_user
24 cmpq TI_addr_limit(%rax),%rcx
25 ja bad_to_user
26 ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \
27 "jmp copy_user_generic_string", \
28 X86_FEATURE_REP_GOOD, \
29 "jmp copy_user_enhanced_fast_string", \
30 X86_FEATURE_ERMS
31ENDPROC(_copy_to_user)
32
33/* Standard copy_from_user with segment limit checking */
34ENTRY(_copy_from_user)
35 GET_THREAD_INFO(%rax)
36 movq %rsi,%rcx
37 addq %rdx,%rcx
38 jc bad_from_user
39 cmpq TI_addr_limit(%rax),%rcx
40 ja bad_from_user
41 ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \
42 "jmp copy_user_generic_string", \
43 X86_FEATURE_REP_GOOD, \
44 "jmp copy_user_enhanced_fast_string", \
45 X86_FEATURE_ERMS
46ENDPROC(_copy_from_user)
47
48 .section .fixup,"ax"
49 /* must zero dest */
50ENTRY(bad_from_user)
51bad_from_user:
52 movl %edx,%ecx
53 xorl %eax,%eax
54 rep
55 stosb
56bad_to_user:
57 movl %edx,%eax
58 ret
59ENDPROC(bad_from_user)
60 .previous
61
62/*
63 * copy_user_generic_unrolled - memory copy with exception handling.
64 * This version is for CPUs like P4 that don't have efficient micro
65 * code for rep movsq
66 *
67 * Input:
68 * rdi destination
69 * rsi source
70 * rdx count
71 *
72 * Output:
73 * eax uncopied bytes or 0 if successful.
74 */
75ENTRY(copy_user_generic_unrolled)
76 ASM_STAC
77 cmpl $8,%edx
78 jb 20f /* less then 8 bytes, go to byte copy loop */
79 ALIGN_DESTINATION
80 movl %edx,%ecx
81 andl $63,%edx
82 shrl $6,%ecx
83 jz 17f
841: movq (%rsi),%r8
852: movq 1*8(%rsi),%r9
863: movq 2*8(%rsi),%r10
874: movq 3*8(%rsi),%r11
885: movq %r8,(%rdi)
896: movq %r9,1*8(%rdi)
907: movq %r10,2*8(%rdi)
918: movq %r11,3*8(%rdi)
929: movq 4*8(%rsi),%r8
9310: movq 5*8(%rsi),%r9
9411: movq 6*8(%rsi),%r10
9512: movq 7*8(%rsi),%r11
9613: movq %r8,4*8(%rdi)
9714: movq %r9,5*8(%rdi)
9815: movq %r10,6*8(%rdi)
9916: movq %r11,7*8(%rdi)
100 leaq 64(%rsi),%rsi
101 leaq 64(%rdi),%rdi
102 decl %ecx
103 jnz 1b
10417: movl %edx,%ecx
105 andl $7,%edx
106 shrl $3,%ecx
107 jz 20f
10818: movq (%rsi),%r8
10919: movq %r8,(%rdi)
110 leaq 8(%rsi),%rsi
111 leaq 8(%rdi),%rdi
112 decl %ecx
113 jnz 18b
11420: andl %edx,%edx
115 jz 23f
116 movl %edx,%ecx
11721: movb (%rsi),%al
11822: movb %al,(%rdi)
119 incq %rsi
120 incq %rdi
121 decl %ecx
122 jnz 21b
12323: xor %eax,%eax
124 ASM_CLAC
125 ret
126
127 .section .fixup,"ax"
12830: shll $6,%ecx
129 addl %ecx,%edx
130 jmp 60f
13140: leal (%rdx,%rcx,8),%edx
132 jmp 60f
13350: movl %ecx,%edx
13460: jmp copy_user_handle_tail /* ecx is zerorest also */
135 .previous
136
137 _ASM_EXTABLE(1b,30b)
138 _ASM_EXTABLE(2b,30b)
139 _ASM_EXTABLE(3b,30b)
140 _ASM_EXTABLE(4b,30b)
141 _ASM_EXTABLE(5b,30b)
142 _ASM_EXTABLE(6b,30b)
143 _ASM_EXTABLE(7b,30b)
144 _ASM_EXTABLE(8b,30b)
145 _ASM_EXTABLE(9b,30b)
146 _ASM_EXTABLE(10b,30b)
147 _ASM_EXTABLE(11b,30b)
148 _ASM_EXTABLE(12b,30b)
149 _ASM_EXTABLE(13b,30b)
150 _ASM_EXTABLE(14b,30b)
151 _ASM_EXTABLE(15b,30b)
152 _ASM_EXTABLE(16b,30b)
153 _ASM_EXTABLE(18b,40b)
154 _ASM_EXTABLE(19b,40b)
155 _ASM_EXTABLE(21b,50b)
156 _ASM_EXTABLE(22b,50b)
157ENDPROC(copy_user_generic_unrolled)
158
159/* Some CPUs run faster using the string copy instructions.
160 * This is also a lot simpler. Use them when possible.
161 *
162 * Only 4GB of copy is supported. This shouldn't be a problem
163 * because the kernel normally only writes from/to page sized chunks
164 * even if user space passed a longer buffer.
165 * And more would be dangerous because both Intel and AMD have
166 * errata with rep movsq > 4GB. If someone feels the need to fix
167 * this please consider this.
168 *
169 * Input:
170 * rdi destination
171 * rsi source
172 * rdx count
173 *
174 * Output:
175 * eax uncopied bytes or 0 if successful.
176 */
177ENTRY(copy_user_generic_string)
178 ASM_STAC
179 cmpl $8,%edx
180 jb 2f /* less than 8 bytes, go to byte copy loop */
181 ALIGN_DESTINATION
182 movl %edx,%ecx
183 shrl $3,%ecx
184 andl $7,%edx
1851: rep
186 movsq
1872: movl %edx,%ecx
1883: rep
189 movsb
190 xorl %eax,%eax
191 ASM_CLAC
192 ret
193
194 .section .fixup,"ax"
19511: leal (%rdx,%rcx,8),%ecx
19612: movl %ecx,%edx /* ecx is zerorest also */
197 jmp copy_user_handle_tail
198 .previous
199
200 _ASM_EXTABLE(1b,11b)
201 _ASM_EXTABLE(3b,12b)
202ENDPROC(copy_user_generic_string)
203
204/*
205 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
206 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
207 *
208 * Input:
209 * rdi destination
210 * rsi source
211 * rdx count
212 *
213 * Output:
214 * eax uncopied bytes or 0 if successful.
215 */
216ENTRY(copy_user_enhanced_fast_string)
217 ASM_STAC
218 movl %edx,%ecx
2191: rep
220 movsb
221 xorl %eax,%eax
222 ASM_CLAC
223 ret
224
225 .section .fixup,"ax"
22612: movl %ecx,%edx /* ecx is zerorest also */
227 jmp copy_user_handle_tail
228 .previous
229
230 _ASM_EXTABLE(1b,12b)
231ENDPROC(copy_user_enhanced_fast_string)
232
233/*
234 * copy_user_nocache - Uncached memory copy with exception handling
235 * This will force destination/source out of cache for more performance.
236 */
237ENTRY(__copy_user_nocache)
238 ASM_STAC
239 cmpl $8,%edx
240 jb 20f /* less then 8 bytes, go to byte copy loop */
241 ALIGN_DESTINATION
242 movl %edx,%ecx
243 andl $63,%edx
244 shrl $6,%ecx
245 jz 17f
2461: movq (%rsi),%r8
2472: movq 1*8(%rsi),%r9
2483: movq 2*8(%rsi),%r10
2494: movq 3*8(%rsi),%r11
2505: movnti %r8,(%rdi)
2516: movnti %r9,1*8(%rdi)
2527: movnti %r10,2*8(%rdi)
2538: movnti %r11,3*8(%rdi)
2549: movq 4*8(%rsi),%r8
25510: movq 5*8(%rsi),%r9
25611: movq 6*8(%rsi),%r10
25712: movq 7*8(%rsi),%r11
25813: movnti %r8,4*8(%rdi)
25914: movnti %r9,5*8(%rdi)
26015: movnti %r10,6*8(%rdi)
26116: movnti %r11,7*8(%rdi)
262 leaq 64(%rsi),%rsi
263 leaq 64(%rdi),%rdi
264 decl %ecx
265 jnz 1b
26617: movl %edx,%ecx
267 andl $7,%edx
268 shrl $3,%ecx
269 jz 20f
27018: movq (%rsi),%r8
27119: movnti %r8,(%rdi)
272 leaq 8(%rsi),%rsi
273 leaq 8(%rdi),%rdi
274 decl %ecx
275 jnz 18b
27620: andl %edx,%edx
277 jz 23f
278 movl %edx,%ecx
27921: movb (%rsi),%al
28022: movb %al,(%rdi)
281 incq %rsi
282 incq %rdi
283 decl %ecx
284 jnz 21b
28523: xorl %eax,%eax
286 ASM_CLAC
287 sfence
288 ret
289
290 .section .fixup,"ax"
29130: shll $6,%ecx
292 addl %ecx,%edx
293 jmp 60f
29440: lea (%rdx,%rcx,8),%rdx
295 jmp 60f
29650: movl %ecx,%edx
29760: sfence
298 jmp copy_user_handle_tail
299 .previous
300
301 _ASM_EXTABLE(1b,30b)
302 _ASM_EXTABLE(2b,30b)
303 _ASM_EXTABLE(3b,30b)
304 _ASM_EXTABLE(4b,30b)
305 _ASM_EXTABLE(5b,30b)
306 _ASM_EXTABLE(6b,30b)
307 _ASM_EXTABLE(7b,30b)
308 _ASM_EXTABLE(8b,30b)
309 _ASM_EXTABLE(9b,30b)
310 _ASM_EXTABLE(10b,30b)
311 _ASM_EXTABLE(11b,30b)
312 _ASM_EXTABLE(12b,30b)
313 _ASM_EXTABLE(13b,30b)
314 _ASM_EXTABLE(14b,30b)
315 _ASM_EXTABLE(15b,30b)
316 _ASM_EXTABLE(16b,30b)
317 _ASM_EXTABLE(18b,40b)
318 _ASM_EXTABLE(19b,40b)
319 _ASM_EXTABLE(21b,50b)
320 _ASM_EXTABLE(22b,50b)
321ENDPROC(__copy_user_nocache)