Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/export.h>
30#include <linux/linkage.h>
31#include <asm/page_types.h>
32#include <asm/errno.h>
33#include <asm/asm-offsets.h>
34#include <asm/thread_info.h>
35#include <asm/asm.h>
36#include <asm/smap.h>
37
38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
39
40.macro check_range size:req
41.if IS_ENABLED(CONFIG_X86_64)
42 movq $0x0123456789abcdef,%rdx
43 1:
44 .pushsection runtime_ptr_USER_PTR_MAX,"a"
45 .long 1b - 8 - .
46 .popsection
47 cmp %rdx, %rax
48 cmova %rdx, %rax
49.else
50 cmp $TASK_SIZE_MAX-\size+1, %eax
51 jae .Lbad_get_user
52 sbb %edx, %edx /* array_index_mask_nospec() */
53 and %edx, %eax
54.endif
55.endm
56
57.macro UACCESS op src dst
581: \op \src,\dst
59 _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
60.endm
61
62
63 .text
64SYM_FUNC_START(__get_user_1)
65 check_range size=1
66 ASM_STAC
67 UACCESS movzbl (%_ASM_AX),%edx
68 xor %eax,%eax
69 ASM_CLAC
70 RET
71SYM_FUNC_END(__get_user_1)
72EXPORT_SYMBOL(__get_user_1)
73
74SYM_FUNC_START(__get_user_2)
75 check_range size=2
76 ASM_STAC
77 UACCESS movzwl (%_ASM_AX),%edx
78 xor %eax,%eax
79 ASM_CLAC
80 RET
81SYM_FUNC_END(__get_user_2)
82EXPORT_SYMBOL(__get_user_2)
83
84SYM_FUNC_START(__get_user_4)
85 check_range size=4
86 ASM_STAC
87 UACCESS movl (%_ASM_AX),%edx
88 xor %eax,%eax
89 ASM_CLAC
90 RET
91SYM_FUNC_END(__get_user_4)
92EXPORT_SYMBOL(__get_user_4)
93
94SYM_FUNC_START(__get_user_8)
95#ifndef CONFIG_X86_64
96 xor %ecx,%ecx
97#endif
98 check_range size=8
99 ASM_STAC
100#ifdef CONFIG_X86_64
101 UACCESS movq (%_ASM_AX),%rdx
102#else
103 UACCESS movl (%_ASM_AX),%edx
104 UACCESS movl 4(%_ASM_AX),%ecx
105#endif
106 xor %eax,%eax
107 ASM_CLAC
108 RET
109SYM_FUNC_END(__get_user_8)
110EXPORT_SYMBOL(__get_user_8)
111
112/* .. and the same for __get_user, just without the range checks */
113SYM_FUNC_START(__get_user_nocheck_1)
114 ASM_STAC
115 ASM_BARRIER_NOSPEC
116 UACCESS movzbl (%_ASM_AX),%edx
117 xor %eax,%eax
118 ASM_CLAC
119 RET
120SYM_FUNC_END(__get_user_nocheck_1)
121EXPORT_SYMBOL(__get_user_nocheck_1)
122
123SYM_FUNC_START(__get_user_nocheck_2)
124 ASM_STAC
125 ASM_BARRIER_NOSPEC
126 UACCESS movzwl (%_ASM_AX),%edx
127 xor %eax,%eax
128 ASM_CLAC
129 RET
130SYM_FUNC_END(__get_user_nocheck_2)
131EXPORT_SYMBOL(__get_user_nocheck_2)
132
133SYM_FUNC_START(__get_user_nocheck_4)
134 ASM_STAC
135 ASM_BARRIER_NOSPEC
136 UACCESS movl (%_ASM_AX),%edx
137 xor %eax,%eax
138 ASM_CLAC
139 RET
140SYM_FUNC_END(__get_user_nocheck_4)
141EXPORT_SYMBOL(__get_user_nocheck_4)
142
143SYM_FUNC_START(__get_user_nocheck_8)
144 ASM_STAC
145 ASM_BARRIER_NOSPEC
146#ifdef CONFIG_X86_64
147 UACCESS movq (%_ASM_AX),%rdx
148#else
149 xor %ecx,%ecx
150 UACCESS movl (%_ASM_AX),%edx
151 UACCESS movl 4(%_ASM_AX),%ecx
152#endif
153 xor %eax,%eax
154 ASM_CLAC
155 RET
156SYM_FUNC_END(__get_user_nocheck_8)
157EXPORT_SYMBOL(__get_user_nocheck_8)
158
159
160SYM_CODE_START_LOCAL(__get_user_handle_exception)
161 ASM_CLAC
162.Lbad_get_user:
163 xor %edx,%edx
164 mov $(-EFAULT),%_ASM_AX
165 RET
166SYM_CODE_END(__get_user_handle_exception)