tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
ARM: Bring back ARMv3 IO and user access code
This partially reverts 357c9c1f07d4546bc3fbc0fd1044d96b114d14ed
(ARM: Remove support for ARMv3 ARM610 and ARM710 CPUs).
Although we only support StrongARM on the RiscPC, we need to keep the
ARMv3 user access code for this platform because the bus does not
understand half-word load/stores.
Reported-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
+816
-3
+20
-3
arch/arm/lib/Makefile
+20
-3
arch/arm/lib/Makefile
···
16
16
call_with_stack.o
17
17
18
18
mmu-y := clear_user.o copy_page.o getuser.o putuser.o
19
-
mmu-y += copy_from_user.o copy_to_user.o
19
+
20
+
# the code in uaccess.S is not preemption safe and
21
+
# probably faster on ARMv3 only
22
+
ifeq ($(CONFIG_PREEMPT),y)
23
+
mmu-y += copy_from_user.o copy_to_user.o
24
+
else
25
+
ifneq ($(CONFIG_CPU_32v3),y)
26
+
mmu-y += copy_from_user.o copy_to_user.o
27
+
else
28
+
mmu-y += uaccess.o
29
+
endif
30
+
endif
20
31
21
32
# using lib_ here won't override already available weak symbols
22
33
obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
23
34
24
-
lib-$(CONFIG_MMU) += $(mmu-y)
25
-
lib-y += io-readsw-armv4.o io-writesw-armv4.o
35
+
lib-$(CONFIG_MMU) += $(mmu-y)
36
+
37
+
ifeq ($(CONFIG_CPU_32v3),y)
38
+
lib-y += io-readsw-armv3.o io-writesw-armv3.o
39
+
else
40
+
lib-y += io-readsw-armv4.o io-writesw-armv4.o
41
+
endif
42
+
26
43
lib-$(CONFIG_ARCH_RPC) += ecard.o io-acorn.o floppydma.o
27
44
lib-$(CONFIG_ARCH_SHARK) += io-shark.o
28
45
+106
arch/arm/lib/io-readsw-armv3.S
+106
arch/arm/lib/io-readsw-armv3.S
···
1
+
/*
2
+
* linux/arch/arm/lib/io-readsw-armv3.S
3
+
*
4
+
* Copyright (C) 1995-2000 Russell King
5
+
*
6
+
* This program is free software; you can redistribute it and/or modify
7
+
* it under the terms of the GNU General Public License version 2 as
8
+
* published by the Free Software Foundation.
9
+
*/
10
+
#include <linux/linkage.h>
11
+
#include <asm/assembler.h>
12
+
13
+
.Linsw_bad_alignment:
14
+
adr r0, .Linsw_bad_align_msg
15
+
mov r2, lr
16
+
b panic
17
+
.Linsw_bad_align_msg:
18
+
.asciz "insw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
19
+
.align
20
+
21
+
.Linsw_align: tst r1, #1
22
+
bne .Linsw_bad_alignment
23
+
24
+
ldr r3, [r0]
25
+
strb r3, [r1], #1
26
+
mov r3, r3, lsr #8
27
+
strb r3, [r1], #1
28
+
29
+
subs r2, r2, #1
30
+
moveq pc, lr
31
+
32
+
ENTRY(__raw_readsw)
33
+
teq r2, #0 @ do we have to check for the zero len?
34
+
moveq pc, lr
35
+
tst r1, #3
36
+
bne .Linsw_align
37
+
38
+
.Linsw_aligned: mov ip, #0xff
39
+
orr ip, ip, ip, lsl #8
40
+
stmfd sp!, {r4, r5, r6, lr}
41
+
42
+
subs r2, r2, #8
43
+
bmi .Lno_insw_8
44
+
45
+
.Linsw_8_lp: ldr r3, [r0]
46
+
and r3, r3, ip
47
+
ldr r4, [r0]
48
+
orr r3, r3, r4, lsl #16
49
+
50
+
ldr r4, [r0]
51
+
and r4, r4, ip
52
+
ldr r5, [r0]
53
+
orr r4, r4, r5, lsl #16
54
+
55
+
ldr r5, [r0]
56
+
and r5, r5, ip
57
+
ldr r6, [r0]
58
+
orr r5, r5, r6, lsl #16
59
+
60
+
ldr r6, [r0]
61
+
and r6, r6, ip
62
+
ldr lr, [r0]
63
+
orr r6, r6, lr, lsl #16
64
+
65
+
stmia r1!, {r3 - r6}
66
+
67
+
subs r2, r2, #8
68
+
bpl .Linsw_8_lp
69
+
70
+
tst r2, #7
71
+
ldmeqfd sp!, {r4, r5, r6, pc}
72
+
73
+
.Lno_insw_8: tst r2, #4
74
+
beq .Lno_insw_4
75
+
76
+
ldr r3, [r0]
77
+
and r3, r3, ip
78
+
ldr r4, [r0]
79
+
orr r3, r3, r4, lsl #16
80
+
81
+
ldr r4, [r0]
82
+
and r4, r4, ip
83
+
ldr r5, [r0]
84
+
orr r4, r4, r5, lsl #16
85
+
86
+
stmia r1!, {r3, r4}
87
+
88
+
.Lno_insw_4: tst r2, #2
89
+
beq .Lno_insw_2
90
+
91
+
ldr r3, [r0]
92
+
and r3, r3, ip
93
+
ldr r4, [r0]
94
+
orr r3, r3, r4, lsl #16
95
+
96
+
str r3, [r1], #4
97
+
98
+
.Lno_insw_2: tst r2, #1
99
+
ldrne r3, [r0]
100
+
strneb r3, [r1], #1
101
+
movne r3, r3, lsr #8
102
+
strneb r3, [r1]
103
+
104
+
ldmfd sp!, {r4, r5, r6, pc}
105
+
106
+
+126
arch/arm/lib/io-writesw-armv3.S
+126
arch/arm/lib/io-writesw-armv3.S
···
1
+
/*
2
+
* linux/arch/arm/lib/io-writesw-armv3.S
3
+
*
4
+
* Copyright (C) 1995-2000 Russell King
5
+
*
6
+
* This program is free software; you can redistribute it and/or modify
7
+
* it under the terms of the GNU General Public License version 2 as
8
+
* published by the Free Software Foundation.
9
+
*/
10
+
#include <linux/linkage.h>
11
+
#include <asm/assembler.h>
12
+
13
+
.Loutsw_bad_alignment:
14
+
adr r0, .Loutsw_bad_align_msg
15
+
mov r2, lr
16
+
b panic
17
+
.Loutsw_bad_align_msg:
18
+
.asciz "outsw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
19
+
.align
20
+
21
+
.Loutsw_align: tst r1, #1
22
+
bne .Loutsw_bad_alignment
23
+
24
+
add r1, r1, #2
25
+
26
+
ldr r3, [r1, #-4]
27
+
mov r3, r3, lsr #16
28
+
orr r3, r3, r3, lsl #16
29
+
str r3, [r0]
30
+
subs r2, r2, #1
31
+
moveq pc, lr
32
+
33
+
ENTRY(__raw_writesw)
34
+
teq r2, #0 @ do we have to check for the zero len?
35
+
moveq pc, lr
36
+
tst r1, #3
37
+
bne .Loutsw_align
38
+
39
+
stmfd sp!, {r4, r5, r6, lr}
40
+
41
+
subs r2, r2, #8
42
+
bmi .Lno_outsw_8
43
+
44
+
.Loutsw_8_lp: ldmia r1!, {r3, r4, r5, r6}
45
+
46
+
mov ip, r3, lsl #16
47
+
orr ip, ip, ip, lsr #16
48
+
str ip, [r0]
49
+
50
+
mov ip, r3, lsr #16
51
+
orr ip, ip, ip, lsl #16
52
+
str ip, [r0]
53
+
54
+
mov ip, r4, lsl #16
55
+
orr ip, ip, ip, lsr #16
56
+
str ip, [r0]
57
+
58
+
mov ip, r4, lsr #16
59
+
orr ip, ip, ip, lsl #16
60
+
str ip, [r0]
61
+
62
+
mov ip, r5, lsl #16
63
+
orr ip, ip, ip, lsr #16
64
+
str ip, [r0]
65
+
66
+
mov ip, r5, lsr #16
67
+
orr ip, ip, ip, lsl #16
68
+
str ip, [r0]
69
+
70
+
mov ip, r6, lsl #16
71
+
orr ip, ip, ip, lsr #16
72
+
str ip, [r0]
73
+
74
+
mov ip, r6, lsr #16
75
+
orr ip, ip, ip, lsl #16
76
+
str ip, [r0]
77
+
78
+
subs r2, r2, #8
79
+
bpl .Loutsw_8_lp
80
+
81
+
tst r2, #7
82
+
ldmeqfd sp!, {r4, r5, r6, pc}
83
+
84
+
.Lno_outsw_8: tst r2, #4
85
+
beq .Lno_outsw_4
86
+
87
+
ldmia r1!, {r3, r4}
88
+
89
+
mov ip, r3, lsl #16
90
+
orr ip, ip, ip, lsr #16
91
+
str ip, [r0]
92
+
93
+
mov ip, r3, lsr #16
94
+
orr ip, ip, ip, lsl #16
95
+
str ip, [r0]
96
+
97
+
mov ip, r4, lsl #16
98
+
orr ip, ip, ip, lsr #16
99
+
str ip, [r0]
100
+
101
+
mov ip, r4, lsr #16
102
+
orr ip, ip, ip, lsl #16
103
+
str ip, [r0]
104
+
105
+
.Lno_outsw_4: tst r2, #2
106
+
beq .Lno_outsw_2
107
+
108
+
ldr r3, [r1], #4
109
+
110
+
mov ip, r3, lsl #16
111
+
orr ip, ip, ip, lsr #16
112
+
str ip, [r0]
113
+
114
+
mov ip, r3, lsr #16
115
+
orr ip, ip, ip, lsl #16
116
+
str ip, [r0]
117
+
118
+
.Lno_outsw_2: tst r2, #1
119
+
120
+
ldrne r3, [r1]
121
+
122
+
movne ip, r3, lsl #16
123
+
orrne ip, ip, ip, lsr #16
124
+
strne ip, [r0]
125
+
126
+
ldmfd sp!, {r4, r5, r6, pc}
+564
arch/arm/lib/uaccess.S
+564
arch/arm/lib/uaccess.S
···
1
+
/*
2
+
* linux/arch/arm/lib/uaccess.S
3
+
*
4
+
* Copyright (C) 1995, 1996,1997,1998 Russell King
5
+
*
6
+
* This program is free software; you can redistribute it and/or modify
7
+
* it under the terms of the GNU General Public License version 2 as
8
+
* published by the Free Software Foundation.
9
+
*
10
+
* Routines to block copy data to/from user memory
11
+
* These are highly optimised both for the 4k page size
12
+
* and for various alignments.
13
+
*/
14
+
#include <linux/linkage.h>
15
+
#include <asm/assembler.h>
16
+
#include <asm/errno.h>
17
+
#include <asm/domain.h>
18
+
19
+
.text
20
+
21
+
#define PAGE_SHIFT 12
22
+
23
+
/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24
+
* Purpose : copy a block to user memory from kernel memory
25
+
* Params : to - user memory
26
+
* : from - kernel memory
27
+
* : n - number of bytes to copy
28
+
* Returns : Number of bytes NOT copied.
29
+
*/
30
+
31
+
.Lc2u_dest_not_aligned:
32
+
rsb ip, ip, #4
33
+
cmp ip, #2
34
+
ldrb r3, [r1], #1
35
+
USER( TUSER( strb) r3, [r0], #1) @ May fault
36
+
ldrgeb r3, [r1], #1
37
+
USER( TUSER( strgeb) r3, [r0], #1) @ May fault
38
+
ldrgtb r3, [r1], #1
39
+
USER( TUSER( strgtb) r3, [r0], #1) @ May fault
40
+
sub r2, r2, ip
41
+
b .Lc2u_dest_aligned
42
+
43
+
ENTRY(__copy_to_user)
44
+
stmfd sp!, {r2, r4 - r7, lr}
45
+
cmp r2, #4
46
+
blt .Lc2u_not_enough
47
+
ands ip, r0, #3
48
+
bne .Lc2u_dest_not_aligned
49
+
.Lc2u_dest_aligned:
50
+
51
+
ands ip, r1, #3
52
+
bne .Lc2u_src_not_aligned
53
+
/*
54
+
* Seeing as there has to be at least 8 bytes to copy, we can
55
+
* copy one word, and force a user-mode page fault...
56
+
*/
57
+
58
+
.Lc2u_0fupi: subs r2, r2, #4
59
+
addmi ip, r2, #4
60
+
bmi .Lc2u_0nowords
61
+
ldr r3, [r1], #4
62
+
USER( TUSER( str) r3, [r0], #4) @ May fault
63
+
mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
64
+
rsb ip, ip, #0
65
+
movs ip, ip, lsr #32 - PAGE_SHIFT
66
+
beq .Lc2u_0fupi
67
+
/*
68
+
* ip = max no. of bytes to copy before needing another "strt" insn
69
+
*/
70
+
cmp r2, ip
71
+
movlt ip, r2
72
+
sub r2, r2, ip
73
+
subs ip, ip, #32
74
+
blt .Lc2u_0rem8lp
75
+
76
+
.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
77
+
stmia r0!, {r3 - r6} @ Shouldnt fault
78
+
ldmia r1!, {r3 - r6}
79
+
subs ip, ip, #32
80
+
stmia r0!, {r3 - r6} @ Shouldnt fault
81
+
bpl .Lc2u_0cpy8lp
82
+
83
+
.Lc2u_0rem8lp: cmn ip, #16
84
+
ldmgeia r1!, {r3 - r6}
85
+
stmgeia r0!, {r3 - r6} @ Shouldnt fault
86
+
tst ip, #8
87
+
ldmneia r1!, {r3 - r4}
88
+
stmneia r0!, {r3 - r4} @ Shouldnt fault
89
+
tst ip, #4
90
+
ldrne r3, [r1], #4
91
+
TUSER( strne) r3, [r0], #4 @ Shouldnt fault
92
+
ands ip, ip, #3
93
+
beq .Lc2u_0fupi
94
+
.Lc2u_0nowords: teq ip, #0
95
+
beq .Lc2u_finished
96
+
.Lc2u_nowords: cmp ip, #2
97
+
ldrb r3, [r1], #1
98
+
USER( TUSER( strb) r3, [r0], #1) @ May fault
99
+
ldrgeb r3, [r1], #1
100
+
USER( TUSER( strgeb) r3, [r0], #1) @ May fault
101
+
ldrgtb r3, [r1], #1
102
+
USER( TUSER( strgtb) r3, [r0], #1) @ May fault
103
+
b .Lc2u_finished
104
+
105
+
.Lc2u_not_enough:
106
+
movs ip, r2
107
+
bne .Lc2u_nowords
108
+
.Lc2u_finished: mov r0, #0
109
+
ldmfd sp!, {r2, r4 - r7, pc}
110
+
111
+
.Lc2u_src_not_aligned:
112
+
bic r1, r1, #3
113
+
ldr r7, [r1], #4
114
+
cmp ip, #2
115
+
bgt .Lc2u_3fupi
116
+
beq .Lc2u_2fupi
117
+
.Lc2u_1fupi: subs r2, r2, #4
118
+
addmi ip, r2, #4
119
+
bmi .Lc2u_1nowords
120
+
mov r3, r7, pull #8
121
+
ldr r7, [r1], #4
122
+
orr r3, r3, r7, push #24
123
+
USER( TUSER( str) r3, [r0], #4) @ May fault
124
+
mov ip, r0, lsl #32 - PAGE_SHIFT
125
+
rsb ip, ip, #0
126
+
movs ip, ip, lsr #32 - PAGE_SHIFT
127
+
beq .Lc2u_1fupi
128
+
cmp r2, ip
129
+
movlt ip, r2
130
+
sub r2, r2, ip
131
+
subs ip, ip, #16
132
+
blt .Lc2u_1rem8lp
133
+
134
+
.Lc2u_1cpy8lp: mov r3, r7, pull #8
135
+
ldmia r1!, {r4 - r7}
136
+
subs ip, ip, #16
137
+
orr r3, r3, r4, push #24
138
+
mov r4, r4, pull #8
139
+
orr r4, r4, r5, push #24
140
+
mov r5, r5, pull #8
141
+
orr r5, r5, r6, push #24
142
+
mov r6, r6, pull #8
143
+
orr r6, r6, r7, push #24
144
+
stmia r0!, {r3 - r6} @ Shouldnt fault
145
+
bpl .Lc2u_1cpy8lp
146
+
147
+
.Lc2u_1rem8lp: tst ip, #8
148
+
movne r3, r7, pull #8
149
+
ldmneia r1!, {r4, r7}
150
+
orrne r3, r3, r4, push #24
151
+
movne r4, r4, pull #8
152
+
orrne r4, r4, r7, push #24
153
+
stmneia r0!, {r3 - r4} @ Shouldnt fault
154
+
tst ip, #4
155
+
movne r3, r7, pull #8
156
+
ldrne r7, [r1], #4
157
+
orrne r3, r3, r7, push #24
158
+
TUSER( strne) r3, [r0], #4 @ Shouldnt fault
159
+
ands ip, ip, #3
160
+
beq .Lc2u_1fupi
161
+
.Lc2u_1nowords: mov r3, r7, get_byte_1
162
+
teq ip, #0
163
+
beq .Lc2u_finished
164
+
cmp ip, #2
165
+
USER( TUSER( strb) r3, [r0], #1) @ May fault
166
+
movge r3, r7, get_byte_2
167
+
USER( TUSER( strgeb) r3, [r0], #1) @ May fault
168
+
movgt r3, r7, get_byte_3
169
+
USER( TUSER( strgtb) r3, [r0], #1) @ May fault
170
+
b .Lc2u_finished
171
+
172
+
.Lc2u_2fupi: subs r2, r2, #4
173
+
addmi ip, r2, #4
174
+
bmi .Lc2u_2nowords
175
+
mov r3, r7, pull #16
176
+
ldr r7, [r1], #4
177
+
orr r3, r3, r7, push #16
178
+
USER( TUSER( str) r3, [r0], #4) @ May fault
179
+
mov ip, r0, lsl #32 - PAGE_SHIFT
180
+
rsb ip, ip, #0
181
+
movs ip, ip, lsr #32 - PAGE_SHIFT
182
+
beq .Lc2u_2fupi
183
+
cmp r2, ip
184
+
movlt ip, r2
185
+
sub r2, r2, ip
186
+
subs ip, ip, #16
187
+
blt .Lc2u_2rem8lp
188
+
189
+
.Lc2u_2cpy8lp: mov r3, r7, pull #16
190
+
ldmia r1!, {r4 - r7}
191
+
subs ip, ip, #16
192
+
orr r3, r3, r4, push #16
193
+
mov r4, r4, pull #16
194
+
orr r4, r4, r5, push #16
195
+
mov r5, r5, pull #16
196
+
orr r5, r5, r6, push #16
197
+
mov r6, r6, pull #16
198
+
orr r6, r6, r7, push #16
199
+
stmia r0!, {r3 - r6} @ Shouldnt fault
200
+
bpl .Lc2u_2cpy8lp
201
+
202
+
.Lc2u_2rem8lp: tst ip, #8
203
+
movne r3, r7, pull #16
204
+
ldmneia r1!, {r4, r7}
205
+
orrne r3, r3, r4, push #16
206
+
movne r4, r4, pull #16
207
+
orrne r4, r4, r7, push #16
208
+
stmneia r0!, {r3 - r4} @ Shouldnt fault
209
+
tst ip, #4
210
+
movne r3, r7, pull #16
211
+
ldrne r7, [r1], #4
212
+
orrne r3, r3, r7, push #16
213
+
TUSER( strne) r3, [r0], #4 @ Shouldnt fault
214
+
ands ip, ip, #3
215
+
beq .Lc2u_2fupi
216
+
.Lc2u_2nowords: mov r3, r7, get_byte_2
217
+
teq ip, #0
218
+
beq .Lc2u_finished
219
+
cmp ip, #2
220
+
USER( TUSER( strb) r3, [r0], #1) @ May fault
221
+
movge r3, r7, get_byte_3
222
+
USER( TUSER( strgeb) r3, [r0], #1) @ May fault
223
+
ldrgtb r3, [r1], #0
224
+
USER( TUSER( strgtb) r3, [r0], #1) @ May fault
225
+
b .Lc2u_finished
226
+
227
+
.Lc2u_3fupi: subs r2, r2, #4
228
+
addmi ip, r2, #4
229
+
bmi .Lc2u_3nowords
230
+
mov r3, r7, pull #24
231
+
ldr r7, [r1], #4
232
+
orr r3, r3, r7, push #8
233
+
USER( TUSER( str) r3, [r0], #4) @ May fault
234
+
mov ip, r0, lsl #32 - PAGE_SHIFT
235
+
rsb ip, ip, #0
236
+
movs ip, ip, lsr #32 - PAGE_SHIFT
237
+
beq .Lc2u_3fupi
238
+
cmp r2, ip
239
+
movlt ip, r2
240
+
sub r2, r2, ip
241
+
subs ip, ip, #16
242
+
blt .Lc2u_3rem8lp
243
+
244
+
.Lc2u_3cpy8lp: mov r3, r7, pull #24
245
+
ldmia r1!, {r4 - r7}
246
+
subs ip, ip, #16
247
+
orr r3, r3, r4, push #8
248
+
mov r4, r4, pull #24
249
+
orr r4, r4, r5, push #8
250
+
mov r5, r5, pull #24
251
+
orr r5, r5, r6, push #8
252
+
mov r6, r6, pull #24
253
+
orr r6, r6, r7, push #8
254
+
stmia r0!, {r3 - r6} @ Shouldnt fault
255
+
bpl .Lc2u_3cpy8lp
256
+
257
+
.Lc2u_3rem8lp: tst ip, #8
258
+
movne r3, r7, pull #24
259
+
ldmneia r1!, {r4, r7}
260
+
orrne r3, r3, r4, push #8
261
+
movne r4, r4, pull #24
262
+
orrne r4, r4, r7, push #8
263
+
stmneia r0!, {r3 - r4} @ Shouldnt fault
264
+
tst ip, #4
265
+
movne r3, r7, pull #24
266
+
ldrne r7, [r1], #4
267
+
orrne r3, r3, r7, push #8
268
+
TUSER( strne) r3, [r0], #4 @ Shouldnt fault
269
+
ands ip, ip, #3
270
+
beq .Lc2u_3fupi
271
+
.Lc2u_3nowords: mov r3, r7, get_byte_3
272
+
teq ip, #0
273
+
beq .Lc2u_finished
274
+
cmp ip, #2
275
+
USER( TUSER( strb) r3, [r0], #1) @ May fault
276
+
ldrgeb r3, [r1], #1
277
+
USER( TUSER( strgeb) r3, [r0], #1) @ May fault
278
+
ldrgtb r3, [r1], #0
279
+
USER( TUSER( strgtb) r3, [r0], #1) @ May fault
280
+
b .Lc2u_finished
281
+
ENDPROC(__copy_to_user)
282
+
283
+
.pushsection .fixup,"ax"
284
+
.align 0
285
+
9001: ldmfd sp!, {r0, r4 - r7, pc}
286
+
.popsection
287
+
288
+
/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289
+
* Purpose : copy a block from user memory to kernel memory
290
+
* Params : to - kernel memory
291
+
* : from - user memory
292
+
* : n - number of bytes to copy
293
+
* Returns : Number of bytes NOT copied.
294
+
*/
295
+
.Lcfu_dest_not_aligned:
296
+
rsb ip, ip, #4
297
+
cmp ip, #2
298
+
USER( TUSER( ldrb) r3, [r1], #1) @ May fault
299
+
strb r3, [r0], #1
300
+
USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
301
+
strgeb r3, [r0], #1
302
+
USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
303
+
strgtb r3, [r0], #1
304
+
sub r2, r2, ip
305
+
b .Lcfu_dest_aligned
306
+
307
+
ENTRY(__copy_from_user)
308
+
stmfd sp!, {r0, r2, r4 - r7, lr}
309
+
cmp r2, #4
310
+
blt .Lcfu_not_enough
311
+
ands ip, r0, #3
312
+
bne .Lcfu_dest_not_aligned
313
+
.Lcfu_dest_aligned:
314
+
ands ip, r1, #3
315
+
bne .Lcfu_src_not_aligned
316
+
317
+
/*
318
+
* Seeing as there has to be at least 8 bytes to copy, we can
319
+
* copy one word, and force a user-mode page fault...
320
+
*/
321
+
322
+
.Lcfu_0fupi: subs r2, r2, #4
323
+
addmi ip, r2, #4
324
+
bmi .Lcfu_0nowords
325
+
USER( TUSER( ldr) r3, [r1], #4)
326
+
str r3, [r0], #4
327
+
mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
328
+
rsb ip, ip, #0
329
+
movs ip, ip, lsr #32 - PAGE_SHIFT
330
+
beq .Lcfu_0fupi
331
+
/*
332
+
* ip = max no. of bytes to copy before needing another "strt" insn
333
+
*/
334
+
cmp r2, ip
335
+
movlt ip, r2
336
+
sub r2, r2, ip
337
+
subs ip, ip, #32
338
+
blt .Lcfu_0rem8lp
339
+
340
+
.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
341
+
stmia r0!, {r3 - r6}
342
+
ldmia r1!, {r3 - r6} @ Shouldnt fault
343
+
subs ip, ip, #32
344
+
stmia r0!, {r3 - r6}
345
+
bpl .Lcfu_0cpy8lp
346
+
347
+
.Lcfu_0rem8lp: cmn ip, #16
348
+
ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349
+
stmgeia r0!, {r3 - r6}
350
+
tst ip, #8
351
+
ldmneia r1!, {r3 - r4} @ Shouldnt fault
352
+
stmneia r0!, {r3 - r4}
353
+
tst ip, #4
354
+
TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault
355
+
strne r3, [r0], #4
356
+
ands ip, ip, #3
357
+
beq .Lcfu_0fupi
358
+
.Lcfu_0nowords: teq ip, #0
359
+
beq .Lcfu_finished
360
+
.Lcfu_nowords: cmp ip, #2
361
+
USER( TUSER( ldrb) r3, [r1], #1) @ May fault
362
+
strb r3, [r0], #1
363
+
USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
364
+
strgeb r3, [r0], #1
365
+
USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
366
+
strgtb r3, [r0], #1
367
+
b .Lcfu_finished
368
+
369
+
.Lcfu_not_enough:
370
+
movs ip, r2
371
+
bne .Lcfu_nowords
372
+
.Lcfu_finished: mov r0, #0
373
+
add sp, sp, #8
374
+
ldmfd sp!, {r4 - r7, pc}
375
+
376
+
.Lcfu_src_not_aligned:
377
+
bic r1, r1, #3
378
+
USER( TUSER( ldr) r7, [r1], #4) @ May fault
379
+
cmp ip, #2
380
+
bgt .Lcfu_3fupi
381
+
beq .Lcfu_2fupi
382
+
.Lcfu_1fupi: subs r2, r2, #4
383
+
addmi ip, r2, #4
384
+
bmi .Lcfu_1nowords
385
+
mov r3, r7, pull #8
386
+
USER( TUSER( ldr) r7, [r1], #4) @ May fault
387
+
orr r3, r3, r7, push #24
388
+
str r3, [r0], #4
389
+
mov ip, r1, lsl #32 - PAGE_SHIFT
390
+
rsb ip, ip, #0
391
+
movs ip, ip, lsr #32 - PAGE_SHIFT
392
+
beq .Lcfu_1fupi
393
+
cmp r2, ip
394
+
movlt ip, r2
395
+
sub r2, r2, ip
396
+
subs ip, ip, #16
397
+
blt .Lcfu_1rem8lp
398
+
399
+
.Lcfu_1cpy8lp: mov r3, r7, pull #8
400
+
ldmia r1!, {r4 - r7} @ Shouldnt fault
401
+
subs ip, ip, #16
402
+
orr r3, r3, r4, push #24
403
+
mov r4, r4, pull #8
404
+
orr r4, r4, r5, push #24
405
+
mov r5, r5, pull #8
406
+
orr r5, r5, r6, push #24
407
+
mov r6, r6, pull #8
408
+
orr r6, r6, r7, push #24
409
+
stmia r0!, {r3 - r6}
410
+
bpl .Lcfu_1cpy8lp
411
+
412
+
.Lcfu_1rem8lp: tst ip, #8
413
+
movne r3, r7, pull #8
414
+
ldmneia r1!, {r4, r7} @ Shouldnt fault
415
+
orrne r3, r3, r4, push #24
416
+
movne r4, r4, pull #8
417
+
orrne r4, r4, r7, push #24
418
+
stmneia r0!, {r3 - r4}
419
+
tst ip, #4
420
+
movne r3, r7, pull #8
421
+
USER( TUSER( ldrne) r7, [r1], #4) @ May fault
422
+
orrne r3, r3, r7, push #24
423
+
strne r3, [r0], #4
424
+
ands ip, ip, #3
425
+
beq .Lcfu_1fupi
426
+
.Lcfu_1nowords: mov r3, r7, get_byte_1
427
+
teq ip, #0
428
+
beq .Lcfu_finished
429
+
cmp ip, #2
430
+
strb r3, [r0], #1
431
+
movge r3, r7, get_byte_2
432
+
strgeb r3, [r0], #1
433
+
movgt r3, r7, get_byte_3
434
+
strgtb r3, [r0], #1
435
+
b .Lcfu_finished
436
+
437
+
.Lcfu_2fupi: subs r2, r2, #4
438
+
addmi ip, r2, #4
439
+
bmi .Lcfu_2nowords
440
+
mov r3, r7, pull #16
441
+
USER( TUSER( ldr) r7, [r1], #4) @ May fault
442
+
orr r3, r3, r7, push #16
443
+
str r3, [r0], #4
444
+
mov ip, r1, lsl #32 - PAGE_SHIFT
445
+
rsb ip, ip, #0
446
+
movs ip, ip, lsr #32 - PAGE_SHIFT
447
+
beq .Lcfu_2fupi
448
+
cmp r2, ip
449
+
movlt ip, r2
450
+
sub r2, r2, ip
451
+
subs ip, ip, #16
452
+
blt .Lcfu_2rem8lp
453
+
454
+
455
+
.Lcfu_2cpy8lp: mov r3, r7, pull #16
456
+
ldmia r1!, {r4 - r7} @ Shouldnt fault
457
+
subs ip, ip, #16
458
+
orr r3, r3, r4, push #16
459
+
mov r4, r4, pull #16
460
+
orr r4, r4, r5, push #16
461
+
mov r5, r5, pull #16
462
+
orr r5, r5, r6, push #16
463
+
mov r6, r6, pull #16
464
+
orr r6, r6, r7, push #16
465
+
stmia r0!, {r3 - r6}
466
+
bpl .Lcfu_2cpy8lp
467
+
468
+
.Lcfu_2rem8lp: tst ip, #8
469
+
movne r3, r7, pull #16
470
+
ldmneia r1!, {r4, r7} @ Shouldnt fault
471
+
orrne r3, r3, r4, push #16
472
+
movne r4, r4, pull #16
473
+
orrne r4, r4, r7, push #16
474
+
stmneia r0!, {r3 - r4}
475
+
tst ip, #4
476
+
movne r3, r7, pull #16
477
+
USER( TUSER( ldrne) r7, [r1], #4) @ May fault
478
+
orrne r3, r3, r7, push #16
479
+
strne r3, [r0], #4
480
+
ands ip, ip, #3
481
+
beq .Lcfu_2fupi
482
+
.Lcfu_2nowords: mov r3, r7, get_byte_2
483
+
teq ip, #0
484
+
beq .Lcfu_finished
485
+
cmp ip, #2
486
+
strb r3, [r0], #1
487
+
movge r3, r7, get_byte_3
488
+
strgeb r3, [r0], #1
489
+
USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault
490
+
strgtb r3, [r0], #1
491
+
b .Lcfu_finished
492
+
493
+
.Lcfu_3fupi: subs r2, r2, #4
494
+
addmi ip, r2, #4
495
+
bmi .Lcfu_3nowords
496
+
mov r3, r7, pull #24
497
+
USER( TUSER( ldr) r7, [r1], #4) @ May fault
498
+
orr r3, r3, r7, push #8
499
+
str r3, [r0], #4
500
+
mov ip, r1, lsl #32 - PAGE_SHIFT
501
+
rsb ip, ip, #0
502
+
movs ip, ip, lsr #32 - PAGE_SHIFT
503
+
beq .Lcfu_3fupi
504
+
cmp r2, ip
505
+
movlt ip, r2
506
+
sub r2, r2, ip
507
+
subs ip, ip, #16
508
+
blt .Lcfu_3rem8lp
509
+
510
+
.Lcfu_3cpy8lp: mov r3, r7, pull #24
511
+
ldmia r1!, {r4 - r7} @ Shouldnt fault
512
+
orr r3, r3, r4, push #8
513
+
mov r4, r4, pull #24
514
+
orr r4, r4, r5, push #8
515
+
mov r5, r5, pull #24
516
+
orr r5, r5, r6, push #8
517
+
mov r6, r6, pull #24
518
+
orr r6, r6, r7, push #8
519
+
stmia r0!, {r3 - r6}
520
+
subs ip, ip, #16
521
+
bpl .Lcfu_3cpy8lp
522
+
523
+
.Lcfu_3rem8lp: tst ip, #8
524
+
movne r3, r7, pull #24
525
+
ldmneia r1!, {r4, r7} @ Shouldnt fault
526
+
orrne r3, r3, r4, push #8
527
+
movne r4, r4, pull #24
528
+
orrne r4, r4, r7, push #8
529
+
stmneia r0!, {r3 - r4}
530
+
tst ip, #4
531
+
movne r3, r7, pull #24
532
+
USER( TUSER( ldrne) r7, [r1], #4) @ May fault
533
+
orrne r3, r3, r7, push #8
534
+
strne r3, [r0], #4
535
+
ands ip, ip, #3
536
+
beq .Lcfu_3fupi
537
+
.Lcfu_3nowords: mov r3, r7, get_byte_3
538
+
teq ip, #0
539
+
beq .Lcfu_finished
540
+
cmp ip, #2
541
+
strb r3, [r0], #1
542
+
USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
543
+
strgeb r3, [r0], #1
544
+
USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
545
+
strgtb r3, [r0], #1
546
+
b .Lcfu_finished
547
+
ENDPROC(__copy_from_user)
548
+
549
+
.pushsection .fixup,"ax"
550
+
.align 0
551
+
/*
552
+
* We took an exception. r0 contains a pointer to
553
+
* the byte not copied.
554
+
*/
555
+
9001: ldr r2, [sp], #4 @ void *to
556
+
sub r2, r0, r2 @ bytes copied
557
+
ldr r1, [sp], #4 @ unsigned long count
558
+
subs r4, r1, r2 @ bytes left to copy
559
+
movne r1, r4
560
+
blne __memzero
561
+
mov r0, r4
562
+
ldmfd sp!, {r4 - r7, pc}
563
+
.popsection
564
+