Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#include <linux/export.h>
7#include <asm/alternative-asm.h>
8#include <asm/asm.h>
9#include <asm/asmmacro.h>
10#include <asm/asm-extable.h>
11#include <asm/cpu.h>
12#include <asm/regdef.h>
13#include <asm/unwind_hints.h>
14
15SYM_FUNC_START(__clear_user)
16#ifdef CONFIG_32BIT
17 b __clear_user_generic
18#else
19 /*
20 * Some CPUs support hardware unaligned access
21 */
22 ALTERNATIVE "b __clear_user_generic", \
23 "b __clear_user_fast", CPU_FEATURE_UAL
24#endif
25SYM_FUNC_END(__clear_user)
26
27EXPORT_SYMBOL(__clear_user)
28
29/*
30 * unsigned long __clear_user_generic(void *addr, size_t size)
31 *
32 * a0: addr
33 * a1: size
34 */
35SYM_FUNC_START(__clear_user_generic)
36 beqz a1, 2f
37
381: st.b zero, a0, 0
39 PTR_ADDI a0, a0, 1
40 PTR_ADDI a1, a1, -1
41 bgtz a1, 1b
42
432: move a0, a1
44 jr ra
45
46 _asm_extable 1b, 2b
47SYM_FUNC_END(__clear_user_generic)
48
49#ifdef CONFIG_64BIT
50/*
51 * unsigned long __clear_user_fast(void *addr, unsigned long size)
52 *
53 * a0: addr
54 * a1: size
55 */
56SYM_FUNC_START(__clear_user_fast)
57 sltui t0, a1, 9
58 bnez t0, .Lsmall
59
60 add.d a2, a0, a1
610: st.d zero, a0, 0
62
63 /* align up address */
64 addi.d a0, a0, 8
65 bstrins.d a0, zero, 2, 0
66
67 addi.d a3, a2, -64
68 bgeu a0, a3, .Llt64
69
70 /* set 64 bytes at a time */
71.Lloop64:
721: st.d zero, a0, 0
732: st.d zero, a0, 8
743: st.d zero, a0, 16
754: st.d zero, a0, 24
765: st.d zero, a0, 32
776: st.d zero, a0, 40
787: st.d zero, a0, 48
798: st.d zero, a0, 56
80 addi.d a0, a0, 64
81 bltu a0, a3, .Lloop64
82
83 /* set the remaining bytes */
84.Llt64:
85 addi.d a3, a2, -32
86 bgeu a0, a3, .Llt32
879: st.d zero, a0, 0
8810: st.d zero, a0, 8
8911: st.d zero, a0, 16
9012: st.d zero, a0, 24
91 addi.d a0, a0, 32
92
93.Llt32:
94 addi.d a3, a2, -16
95 bgeu a0, a3, .Llt16
9613: st.d zero, a0, 0
9714: st.d zero, a0, 8
98 addi.d a0, a0, 16
99
100.Llt16:
101 addi.d a3, a2, -8
102 bgeu a0, a3, .Llt8
10315: st.d zero, a0, 0
104 addi.d a0, a0, 8
105
106.Llt8:
10716: st.d zero, a2, -8
108
109 /* return */
110 move a0, zero
111 jr ra
112
113 .align 4
114.Lsmall:
115 pcaddi t0, 4
116 slli.d a2, a1, 4
117 add.d t0, t0, a2
118 jr t0
119
120 .align 4
121 move a0, zero
122 jr ra
123
124 .align 4
12517: st.b zero, a0, 0
126 move a0, zero
127 jr ra
128
129 .align 4
13018: st.h zero, a0, 0
131 move a0, zero
132 jr ra
133
134 .align 4
13519: st.h zero, a0, 0
13620: st.b zero, a0, 2
137 move a0, zero
138 jr ra
139
140 .align 4
14121: st.w zero, a0, 0
142 move a0, zero
143 jr ra
144
145 .align 4
14622: st.w zero, a0, 0
14723: st.b zero, a0, 4
148 move a0, zero
149 jr ra
150
151 .align 4
15224: st.w zero, a0, 0
15325: st.h zero, a0, 4
154 move a0, zero
155 jr ra
156
157 .align 4
15826: st.w zero, a0, 0
15927: st.w zero, a0, 3
160 move a0, zero
161 jr ra
162
163 .align 4
16428: st.d zero, a0, 0
165 move a0, zero
166 jr ra
167
168 /* fixup and ex_table */
169.Llarge_fixup:
170 sub.d a1, a2, a0
171
172.Lsmall_fixup:
17329: st.b zero, a0, 0
174 addi.d a0, a0, 1
175 addi.d a1, a1, -1
176 bgt a1, zero, 29b
177
178.Lexit:
179 move a0, a1
180 jr ra
181
182 _asm_extable 0b, .Lsmall_fixup
183 _asm_extable 1b, .Llarge_fixup
184 _asm_extable 2b, .Llarge_fixup
185 _asm_extable 3b, .Llarge_fixup
186 _asm_extable 4b, .Llarge_fixup
187 _asm_extable 5b, .Llarge_fixup
188 _asm_extable 6b, .Llarge_fixup
189 _asm_extable 7b, .Llarge_fixup
190 _asm_extable 8b, .Llarge_fixup
191 _asm_extable 9b, .Llarge_fixup
192 _asm_extable 10b, .Llarge_fixup
193 _asm_extable 11b, .Llarge_fixup
194 _asm_extable 12b, .Llarge_fixup
195 _asm_extable 13b, .Llarge_fixup
196 _asm_extable 14b, .Llarge_fixup
197 _asm_extable 15b, .Llarge_fixup
198 _asm_extable 16b, .Llarge_fixup
199 _asm_extable 17b, .Lexit
200 _asm_extable 18b, .Lsmall_fixup
201 _asm_extable 19b, .Lsmall_fixup
202 _asm_extable 20b, .Lsmall_fixup
203 _asm_extable 21b, .Lsmall_fixup
204 _asm_extable 22b, .Lsmall_fixup
205 _asm_extable 23b, .Lsmall_fixup
206 _asm_extable 24b, .Lsmall_fixup
207 _asm_extable 25b, .Lsmall_fixup
208 _asm_extable 26b, .Lsmall_fixup
209 _asm_extable 27b, .Lsmall_fixup
210 _asm_extable 28b, .Lsmall_fixup
211 _asm_extable 29b, .Lexit
212SYM_FUNC_END(__clear_user_fast)
213
214STACK_FRAME_NON_STANDARD __clear_user_fast
215#endif