Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#include <linux/export.h>
7#include <asm/alternative-asm.h>
8#include <asm/asm.h>
9#include <asm/asmmacro.h>
10#include <asm/asm-extable.h>
11#include <asm/cpu.h>
12#include <asm/regdef.h>
13#include <asm/unwind_hints.h>
14
15SYM_FUNC_START(__copy_user)
16#ifdef CONFIG_32BIT
17 b __copy_user_generic
18#else
19 /*
20 * Some CPUs support hardware unaligned access
21 */
22 ALTERNATIVE "b __copy_user_generic", \
23 "b __copy_user_fast", CPU_FEATURE_UAL
24#endif
25SYM_FUNC_END(__copy_user)
26
27EXPORT_SYMBOL(__copy_user)
28
29/*
30 * unsigned long __copy_user_generic(void *to, const void *from, size_t n)
31 *
32 * a0: to
33 * a1: from
34 * a2: n
35 */
36SYM_FUNC_START(__copy_user_generic)
37 beqz a2, 3f
38
391: ld.b t0, a1, 0
402: st.b t0, a0, 0
41 PTR_ADDI a0, a0, 1
42 PTR_ADDI a1, a1, 1
43 PTR_ADDI a2, a2, -1
44 bgtz a2, 1b
45
463: move a0, a2
47 jr ra
48
49 _asm_extable 1b, 3b
50 _asm_extable 2b, 3b
51SYM_FUNC_END(__copy_user_generic)
52
53#ifdef CONFIG_64BIT
54/*
55 * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n)
56 *
57 * a0: to
58 * a1: from
59 * a2: n
60 */
61SYM_FUNC_START(__copy_user_fast)
62 sltui t0, a2, 9
63 bnez t0, .Lsmall
64
650: ld.d t0, a1, 0
661: st.d t0, a0, 0
67 add.d a3, a1, a2
68 add.d a2, a0, a2
69
70 /* align up destination address */
71 andi t1, a0, 7
72 sub.d t0, zero, t1
73 addi.d t0, t0, 8
74 add.d a1, a1, t0
75 add.d a0, a0, t0
76
77 addi.d a4, a3, -64
78 bgeu a1, a4, .Llt64
79
80 /* copy 64 bytes at a time */
81.Lloop64:
822: ld.d t0, a1, 0
833: ld.d t1, a1, 8
844: ld.d t2, a1, 16
855: ld.d t3, a1, 24
866: ld.d t4, a1, 32
877: ld.d t5, a1, 40
888: ld.d t6, a1, 48
899: ld.d t7, a1, 56
9010: st.d t0, a0, 0
9111: st.d t1, a0, 8
9212: st.d t2, a0, 16
9313: st.d t3, a0, 24
9414: st.d t4, a0, 32
9515: st.d t5, a0, 40
9616: st.d t6, a0, 48
9717: st.d t7, a0, 56
98 addi.d a1, a1, 64
99 addi.d a0, a0, 64
100 bltu a1, a4, .Lloop64
101
102 /* copy the remaining bytes */
103.Llt64:
104 addi.d a4, a3, -32
105 bgeu a1, a4, .Llt32
10618: ld.d t0, a1, 0
10719: ld.d t1, a1, 8
10820: ld.d t2, a1, 16
10921: ld.d t3, a1, 24
11022: st.d t0, a0, 0
11123: st.d t1, a0, 8
11224: st.d t2, a0, 16
11325: st.d t3, a0, 24
114 addi.d a1, a1, 32
115 addi.d a0, a0, 32
116
117.Llt32:
118 addi.d a4, a3, -16
119 bgeu a1, a4, .Llt16
12026: ld.d t0, a1, 0
12127: ld.d t1, a1, 8
12228: st.d t0, a0, 0
12329: st.d t1, a0, 8
124 addi.d a1, a1, 16
125 addi.d a0, a0, 16
126
127.Llt16:
128 addi.d a4, a3, -8
129 bgeu a1, a4, .Llt8
13030: ld.d t0, a1, 0
13131: st.d t0, a0, 0
132 addi.d a1, a1, 8
133 addi.d a0, a0, 8
134
135.Llt8:
13632: ld.d t0, a3, -8
13733: st.d t0, a2, -8
138
139 /* return */
140 move a0, zero
141 jr ra
142
143 .align 5
144.Lsmall:
145 pcaddi t0, 8
146 slli.d a3, a2, 5
147 add.d t0, t0, a3
148 jr t0
149
150 .align 5
151 move a0, zero
152 jr ra
153
154 .align 5
15534: ld.b t0, a1, 0
15635: st.b t0, a0, 0
157 move a0, zero
158 jr ra
159
160 .align 5
16136: ld.h t0, a1, 0
16237: st.h t0, a0, 0
163 move a0, zero
164 jr ra
165
166 .align 5
16738: ld.h t0, a1, 0
16839: ld.b t1, a1, 2
16940: st.h t0, a0, 0
17041: st.b t1, a0, 2
171 move a0, zero
172 jr ra
173
174 .align 5
17542: ld.w t0, a1, 0
17643: st.w t0, a0, 0
177 move a0, zero
178 jr ra
179
180 .align 5
18144: ld.w t0, a1, 0
18245: ld.b t1, a1, 4
18346: st.w t0, a0, 0
18447: st.b t1, a0, 4
185 move a0, zero
186 jr ra
187
188 .align 5
18948: ld.w t0, a1, 0
19049: ld.h t1, a1, 4
19150: st.w t0, a0, 0
19251: st.h t1, a0, 4
193 move a0, zero
194 jr ra
195
196 .align 5
19752: ld.w t0, a1, 0
19853: ld.w t1, a1, 3
19954: st.w t0, a0, 0
20055: st.w t1, a0, 3
201 move a0, zero
202 jr ra
203
204 .align 5
20556: ld.d t0, a1, 0
20657: st.d t0, a0, 0
207 move a0, zero
208 jr ra
209
210 /* fixup and ex_table */
211.Llarge_fixup:
212 sub.d a2, a2, a0
213
214.Lsmall_fixup:
21558: ld.b t0, a1, 0
21659: st.b t0, a0, 0
217 addi.d a0, a0, 1
218 addi.d a1, a1, 1
219 addi.d a2, a2, -1
220 bgt a2, zero, 58b
221
222.Lexit:
223 move a0, a2
224 jr ra
225
226 _asm_extable 0b, .Lsmall_fixup
227 _asm_extable 1b, .Lsmall_fixup
228 _asm_extable 2b, .Llarge_fixup
229 _asm_extable 3b, .Llarge_fixup
230 _asm_extable 4b, .Llarge_fixup
231 _asm_extable 5b, .Llarge_fixup
232 _asm_extable 6b, .Llarge_fixup
233 _asm_extable 7b, .Llarge_fixup
234 _asm_extable 8b, .Llarge_fixup
235 _asm_extable 9b, .Llarge_fixup
236 _asm_extable 10b, .Llarge_fixup
237 _asm_extable 11b, .Llarge_fixup
238 _asm_extable 12b, .Llarge_fixup
239 _asm_extable 13b, .Llarge_fixup
240 _asm_extable 14b, .Llarge_fixup
241 _asm_extable 15b, .Llarge_fixup
242 _asm_extable 16b, .Llarge_fixup
243 _asm_extable 17b, .Llarge_fixup
244 _asm_extable 18b, .Llarge_fixup
245 _asm_extable 19b, .Llarge_fixup
246 _asm_extable 20b, .Llarge_fixup
247 _asm_extable 21b, .Llarge_fixup
248 _asm_extable 22b, .Llarge_fixup
249 _asm_extable 23b, .Llarge_fixup
250 _asm_extable 24b, .Llarge_fixup
251 _asm_extable 25b, .Llarge_fixup
252 _asm_extable 26b, .Llarge_fixup
253 _asm_extable 27b, .Llarge_fixup
254 _asm_extable 28b, .Llarge_fixup
255 _asm_extable 29b, .Llarge_fixup
256 _asm_extable 30b, .Llarge_fixup
257 _asm_extable 31b, .Llarge_fixup
258 _asm_extable 32b, .Llarge_fixup
259 _asm_extable 33b, .Llarge_fixup
260 _asm_extable 34b, .Lexit
261 _asm_extable 35b, .Lexit
262 _asm_extable 36b, .Lsmall_fixup
263 _asm_extable 37b, .Lsmall_fixup
264 _asm_extable 38b, .Lsmall_fixup
265 _asm_extable 39b, .Lsmall_fixup
266 _asm_extable 40b, .Lsmall_fixup
267 _asm_extable 41b, .Lsmall_fixup
268 _asm_extable 42b, .Lsmall_fixup
269 _asm_extable 43b, .Lsmall_fixup
270 _asm_extable 44b, .Lsmall_fixup
271 _asm_extable 45b, .Lsmall_fixup
272 _asm_extable 46b, .Lsmall_fixup
273 _asm_extable 47b, .Lsmall_fixup
274 _asm_extable 48b, .Lsmall_fixup
275 _asm_extable 49b, .Lsmall_fixup
276 _asm_extable 50b, .Lsmall_fixup
277 _asm_extable 51b, .Lsmall_fixup
278 _asm_extable 52b, .Lsmall_fixup
279 _asm_extable 53b, .Lsmall_fixup
280 _asm_extable 54b, .Lsmall_fixup
281 _asm_extable 55b, .Lsmall_fixup
282 _asm_extable 56b, .Lsmall_fixup
283 _asm_extable 57b, .Lsmall_fixup
284 _asm_extable 58b, .Lexit
285 _asm_extable 59b, .Lexit
286SYM_FUNC_END(__copy_user_fast)
287
288STACK_FRAME_NON_STANDARD __copy_user_fast
289#endif