Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/export.h>
30#include <linux/linkage.h>
31#include <asm/page_types.h>
32#include <asm/errno.h>
33#include <asm/asm-offsets.h>
34#include <asm/thread_info.h>
35#include <asm/asm.h>
36#include <asm/smap.h>
37
38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
39
40.macro check_range size:req
41.if IS_ENABLED(CONFIG_X86_64)
42 mov %rax, %rdx
43 sar $63, %rdx
44 or %rdx, %rax
45.else
46 cmp $TASK_SIZE_MAX-\size+1, %eax
47.if \size != 8
48 jae .Lbad_get_user
49.else
50 jae .Lbad_get_user_8
51.endif
52 sbb %edx, %edx /* array_index_mask_nospec() */
53 and %edx, %eax
54.endif
55.endm
56
57 .text
58SYM_FUNC_START(__get_user_1)
59 check_range size=1
60 ASM_STAC
611: movzbl (%_ASM_AX),%edx
62 xor %eax,%eax
63 ASM_CLAC
64 RET
65SYM_FUNC_END(__get_user_1)
66EXPORT_SYMBOL(__get_user_1)
67
68SYM_FUNC_START(__get_user_2)
69 check_range size=2
70 ASM_STAC
712: movzwl (%_ASM_AX),%edx
72 xor %eax,%eax
73 ASM_CLAC
74 RET
75SYM_FUNC_END(__get_user_2)
76EXPORT_SYMBOL(__get_user_2)
77
78SYM_FUNC_START(__get_user_4)
79 check_range size=4
80 ASM_STAC
813: movl (%_ASM_AX),%edx
82 xor %eax,%eax
83 ASM_CLAC
84 RET
85SYM_FUNC_END(__get_user_4)
86EXPORT_SYMBOL(__get_user_4)
87
88SYM_FUNC_START(__get_user_8)
89 check_range size=8
90 ASM_STAC
91#ifdef CONFIG_X86_64
924: movq (%_ASM_AX),%rdx
93#else
944: movl (%_ASM_AX),%edx
955: movl 4(%_ASM_AX),%ecx
96#endif
97 xor %eax,%eax
98 ASM_CLAC
99 RET
100SYM_FUNC_END(__get_user_8)
101EXPORT_SYMBOL(__get_user_8)
102
103/* .. and the same for __get_user, just without the range checks */
104SYM_FUNC_START(__get_user_nocheck_1)
105 ASM_STAC
106 ASM_BARRIER_NOSPEC
1076: movzbl (%_ASM_AX),%edx
108 xor %eax,%eax
109 ASM_CLAC
110 RET
111SYM_FUNC_END(__get_user_nocheck_1)
112EXPORT_SYMBOL(__get_user_nocheck_1)
113
114SYM_FUNC_START(__get_user_nocheck_2)
115 ASM_STAC
116 ASM_BARRIER_NOSPEC
1177: movzwl (%_ASM_AX),%edx
118 xor %eax,%eax
119 ASM_CLAC
120 RET
121SYM_FUNC_END(__get_user_nocheck_2)
122EXPORT_SYMBOL(__get_user_nocheck_2)
123
124SYM_FUNC_START(__get_user_nocheck_4)
125 ASM_STAC
126 ASM_BARRIER_NOSPEC
1278: movl (%_ASM_AX),%edx
128 xor %eax,%eax
129 ASM_CLAC
130 RET
131SYM_FUNC_END(__get_user_nocheck_4)
132EXPORT_SYMBOL(__get_user_nocheck_4)
133
134SYM_FUNC_START(__get_user_nocheck_8)
135 ASM_STAC
136 ASM_BARRIER_NOSPEC
137#ifdef CONFIG_X86_64
1389: movq (%_ASM_AX),%rdx
139#else
1409: movl (%_ASM_AX),%edx
14110: movl 4(%_ASM_AX),%ecx
142#endif
143 xor %eax,%eax
144 ASM_CLAC
145 RET
146SYM_FUNC_END(__get_user_nocheck_8)
147EXPORT_SYMBOL(__get_user_nocheck_8)
148
149
150SYM_CODE_START_LOCAL(__get_user_handle_exception)
151 ASM_CLAC
152.Lbad_get_user:
153 xor %edx,%edx
154 mov $(-EFAULT),%_ASM_AX
155 RET
156SYM_CODE_END(__get_user_handle_exception)
157
158#ifdef CONFIG_X86_32
159SYM_CODE_START_LOCAL(__get_user_8_handle_exception)
160 ASM_CLAC
161.Lbad_get_user_8:
162 xor %edx,%edx
163 xor %ecx,%ecx
164 mov $(-EFAULT),%_ASM_AX
165 RET
166SYM_CODE_END(__get_user_8_handle_exception)
167#endif
168
169/* get_user */
170 _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
171 _ASM_EXTABLE_UA(2b, __get_user_handle_exception)
172 _ASM_EXTABLE_UA(3b, __get_user_handle_exception)
173#ifdef CONFIG_X86_64
174 _ASM_EXTABLE_UA(4b, __get_user_handle_exception)
175#else
176 _ASM_EXTABLE_UA(4b, __get_user_8_handle_exception)
177 _ASM_EXTABLE_UA(5b, __get_user_8_handle_exception)
178#endif
179
180/* __get_user */
181 _ASM_EXTABLE_UA(6b, __get_user_handle_exception)
182 _ASM_EXTABLE_UA(7b, __get_user_handle_exception)
183 _ASM_EXTABLE_UA(8b, __get_user_handle_exception)
184#ifdef CONFIG_X86_64
185 _ASM_EXTABLE_UA(9b, __get_user_handle_exception)
186#else
187 _ASM_EXTABLE_UA(9b, __get_user_8_handle_exception)
188 _ASM_EXTABLE_UA(10b, __get_user_8_handle_exception)
189#endif