tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * linux/arch/arm/lib/uaccess.S
3 *
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
13 */
14#include <linux/linkage.h>
15#include <asm/assembler.h>
16#include <asm/errno.h>
17
18 .text
19
20#define PAGE_SHIFT 12
21
22/* Prototype: int __arch_copy_to_user(void *to, const char *from, size_t n)
23 * Purpose : copy a block to user memory from kernel memory
24 * Params : to - user memory
25 * : from - kernel memory
26 * : n - number of bytes to copy
27 * Returns : Number of bytes NOT copied.
28 */
29
30.Lc2u_dest_not_aligned:
31 rsb ip, ip, #4
32 cmp ip, #2
33 ldrb r3, [r1], #1
34USER( strbt r3, [r0], #1) @ May fault
35 ldrgeb r3, [r1], #1
36USER( strgebt r3, [r0], #1) @ May fault
37 ldrgtb r3, [r1], #1
38USER( strgtbt r3, [r0], #1) @ May fault
39 sub r2, r2, ip
40 b .Lc2u_dest_aligned
41
42ENTRY(__arch_copy_to_user)
43 stmfd sp!, {r2, r4 - r7, lr}
44 cmp r2, #4
45 blt .Lc2u_not_enough
46 ands ip, r0, #3
47 bne .Lc2u_dest_not_aligned
48.Lc2u_dest_aligned:
49
50 ands ip, r1, #3
51 bne .Lc2u_src_not_aligned
52/*
53 * Seeing as there has to be at least 8 bytes to copy, we can
54 * copy one word, and force a user-mode page fault...
55 */
56
57.Lc2u_0fupi: subs r2, r2, #4
58 addmi ip, r2, #4
59 bmi .Lc2u_0nowords
60 ldr r3, [r1], #4
61USER( strt r3, [r0], #4) @ May fault
62 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
63 rsb ip, ip, #0
64 movs ip, ip, lsr #32 - PAGE_SHIFT
65 beq .Lc2u_0fupi
66/*
67 * ip = max no. of bytes to copy before needing another "strt" insn
68 */
69 cmp r2, ip
70 movlt ip, r2
71 sub r2, r2, ip
72 subs ip, ip, #32
73 blt .Lc2u_0rem8lp
74
75.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
76 stmia r0!, {r3 - r6} @ Shouldnt fault
77 ldmia r1!, {r3 - r6}
78 subs ip, ip, #32
79 stmia r0!, {r3 - r6} @ Shouldnt fault
80 bpl .Lc2u_0cpy8lp
81
82.Lc2u_0rem8lp: cmn ip, #16
83 ldmgeia r1!, {r3 - r6}
84 stmgeia r0!, {r3 - r6} @ Shouldnt fault
85 tst ip, #8
86 ldmneia r1!, {r3 - r4}
87 stmneia r0!, {r3 - r4} @ Shouldnt fault
88 tst ip, #4
89 ldrne r3, [r1], #4
90 strnet r3, [r0], #4 @ Shouldnt fault
91 ands ip, ip, #3
92 beq .Lc2u_0fupi
93.Lc2u_0nowords: teq ip, #0
94 beq .Lc2u_finished
95.Lc2u_nowords: cmp ip, #2
96 ldrb r3, [r1], #1
97USER( strbt r3, [r0], #1) @ May fault
98 ldrgeb r3, [r1], #1
99USER( strgebt r3, [r0], #1) @ May fault
100 ldrgtb r3, [r1], #1
101USER( strgtbt r3, [r0], #1) @ May fault
102 b .Lc2u_finished
103
104.Lc2u_not_enough:
105 movs ip, r2
106 bne .Lc2u_nowords
107.Lc2u_finished: mov r0, #0
108 LOADREGS(fd,sp!,{r2, r4 - r7, pc})
109
110.Lc2u_src_not_aligned:
111 bic r1, r1, #3
112 ldr r7, [r1], #4
113 cmp ip, #2
114 bgt .Lc2u_3fupi
115 beq .Lc2u_2fupi
116.Lc2u_1fupi: subs r2, r2, #4
117 addmi ip, r2, #4
118 bmi .Lc2u_1nowords
119 mov r3, r7, pull #8
120 ldr r7, [r1], #4
121 orr r3, r3, r7, push #24
122USER( strt r3, [r0], #4) @ May fault
123 mov ip, r0, lsl #32 - PAGE_SHIFT
124 rsb ip, ip, #0
125 movs ip, ip, lsr #32 - PAGE_SHIFT
126 beq .Lc2u_1fupi
127 cmp r2, ip
128 movlt ip, r2
129 sub r2, r2, ip
130 subs ip, ip, #16
131 blt .Lc2u_1rem8lp
132
133.Lc2u_1cpy8lp: mov r3, r7, pull #8
134 ldmia r1!, {r4 - r7}
135 subs ip, ip, #16
136 orr r3, r3, r4, push #24
137 mov r4, r4, pull #8
138 orr r4, r4, r5, push #24
139 mov r5, r5, pull #8
140 orr r5, r5, r6, push #24
141 mov r6, r6, pull #8
142 orr r6, r6, r7, push #24
143 stmia r0!, {r3 - r6} @ Shouldnt fault
144 bpl .Lc2u_1cpy8lp
145
146.Lc2u_1rem8lp: tst ip, #8
147 movne r3, r7, pull #8
148 ldmneia r1!, {r4, r7}
149 orrne r3, r3, r4, push #24
150 movne r4, r4, pull #8
151 orrne r4, r4, r7, push #24
152 stmneia r0!, {r3 - r4} @ Shouldnt fault
153 tst ip, #4
154 movne r3, r7, pull #8
155 ldrne r7, [r1], #4
156 orrne r3, r3, r7, push #24
157 strnet r3, [r0], #4 @ Shouldnt fault
158 ands ip, ip, #3
159 beq .Lc2u_1fupi
160.Lc2u_1nowords: mov r3, r7, get_byte_1
161 teq ip, #0
162 beq .Lc2u_finished
163 cmp ip, #2
164USER( strbt r3, [r0], #1) @ May fault
165 movge r3, r7, get_byte_2
166USER( strgebt r3, [r0], #1) @ May fault
167 movgt r3, r7, get_byte_3
168USER( strgtbt r3, [r0], #1) @ May fault
169 b .Lc2u_finished
170
171.Lc2u_2fupi: subs r2, r2, #4
172 addmi ip, r2, #4
173 bmi .Lc2u_2nowords
174 mov r3, r7, pull #16
175 ldr r7, [r1], #4
176 orr r3, r3, r7, push #16
177USER( strt r3, [r0], #4) @ May fault
178 mov ip, r0, lsl #32 - PAGE_SHIFT
179 rsb ip, ip, #0
180 movs ip, ip, lsr #32 - PAGE_SHIFT
181 beq .Lc2u_2fupi
182 cmp r2, ip
183 movlt ip, r2
184 sub r2, r2, ip
185 subs ip, ip, #16
186 blt .Lc2u_2rem8lp
187
188.Lc2u_2cpy8lp: mov r3, r7, pull #16
189 ldmia r1!, {r4 - r7}
190 subs ip, ip, #16
191 orr r3, r3, r4, push #16
192 mov r4, r4, pull #16
193 orr r4, r4, r5, push #16
194 mov r5, r5, pull #16
195 orr r5, r5, r6, push #16
196 mov r6, r6, pull #16
197 orr r6, r6, r7, push #16
198 stmia r0!, {r3 - r6} @ Shouldnt fault
199 bpl .Lc2u_2cpy8lp
200
201.Lc2u_2rem8lp: tst ip, #8
202 movne r3, r7, pull #16
203 ldmneia r1!, {r4, r7}
204 orrne r3, r3, r4, push #16
205 movne r4, r4, pull #16
206 orrne r4, r4, r7, push #16
207 stmneia r0!, {r3 - r4} @ Shouldnt fault
208 tst ip, #4
209 movne r3, r7, pull #16
210 ldrne r7, [r1], #4
211 orrne r3, r3, r7, push #16
212 strnet r3, [r0], #4 @ Shouldnt fault
213 ands ip, ip, #3
214 beq .Lc2u_2fupi
215.Lc2u_2nowords: mov r3, r7, get_byte_2
216 teq ip, #0
217 beq .Lc2u_finished
218 cmp ip, #2
219USER( strbt r3, [r0], #1) @ May fault
220 movge r3, r7, get_byte_3
221USER( strgebt r3, [r0], #1) @ May fault
222 ldrgtb r3, [r1], #0
223USER( strgtbt r3, [r0], #1) @ May fault
224 b .Lc2u_finished
225
226.Lc2u_3fupi: subs r2, r2, #4
227 addmi ip, r2, #4
228 bmi .Lc2u_3nowords
229 mov r3, r7, pull #24
230 ldr r7, [r1], #4
231 orr r3, r3, r7, push #8
232USER( strt r3, [r0], #4) @ May fault
233 mov ip, r0, lsl #32 - PAGE_SHIFT
234 rsb ip, ip, #0
235 movs ip, ip, lsr #32 - PAGE_SHIFT
236 beq .Lc2u_3fupi
237 cmp r2, ip
238 movlt ip, r2
239 sub r2, r2, ip
240 subs ip, ip, #16
241 blt .Lc2u_3rem8lp
242
243.Lc2u_3cpy8lp: mov r3, r7, pull #24
244 ldmia r1!, {r4 - r7}
245 subs ip, ip, #16
246 orr r3, r3, r4, push #8
247 mov r4, r4, pull #24
248 orr r4, r4, r5, push #8
249 mov r5, r5, pull #24
250 orr r5, r5, r6, push #8
251 mov r6, r6, pull #24
252 orr r6, r6, r7, push #8
253 stmia r0!, {r3 - r6} @ Shouldnt fault
254 bpl .Lc2u_3cpy8lp
255
256.Lc2u_3rem8lp: tst ip, #8
257 movne r3, r7, pull #24
258 ldmneia r1!, {r4, r7}
259 orrne r3, r3, r4, push #8
260 movne r4, r4, pull #24
261 orrne r4, r4, r7, push #8
262 stmneia r0!, {r3 - r4} @ Shouldnt fault
263 tst ip, #4
264 movne r3, r7, pull #24
265 ldrne r7, [r1], #4
266 orrne r3, r3, r7, push #8
267 strnet r3, [r0], #4 @ Shouldnt fault
268 ands ip, ip, #3
269 beq .Lc2u_3fupi
270.Lc2u_3nowords: mov r3, r7, get_byte_3
271 teq ip, #0
272 beq .Lc2u_finished
273 cmp ip, #2
274USER( strbt r3, [r0], #1) @ May fault
275 ldrgeb r3, [r1], #1
276USER( strgebt r3, [r0], #1) @ May fault
277 ldrgtb r3, [r1], #0
278USER( strgtbt r3, [r0], #1) @ May fault
279 b .Lc2u_finished
280
281 .section .fixup,"ax"
282 .align 0
2839001: LOADREGS(fd,sp!, {r0, r4 - r7, pc})
284 .previous
285
286/* Prototype: unsigned long __arch_copy_from_user(void *to,const void *from,unsigned long n);
287 * Purpose : copy a block from user memory to kernel memory
288 * Params : to - kernel memory
289 * : from - user memory
290 * : n - number of bytes to copy
291 * Returns : Number of bytes NOT copied.
292 */
293.Lcfu_dest_not_aligned:
294 rsb ip, ip, #4
295 cmp ip, #2
296USER( ldrbt r3, [r1], #1) @ May fault
297 strb r3, [r0], #1
298USER( ldrgebt r3, [r1], #1) @ May fault
299 strgeb r3, [r0], #1
300USER( ldrgtbt r3, [r1], #1) @ May fault
301 strgtb r3, [r0], #1
302 sub r2, r2, ip
303 b .Lcfu_dest_aligned
304
305ENTRY(__arch_copy_from_user)
306 stmfd sp!, {r0, r2, r4 - r7, lr}
307 cmp r2, #4
308 blt .Lcfu_not_enough
309 ands ip, r0, #3
310 bne .Lcfu_dest_not_aligned
311.Lcfu_dest_aligned:
312 ands ip, r1, #3
313 bne .Lcfu_src_not_aligned
314
315/*
316 * Seeing as there has to be at least 8 bytes to copy, we can
317 * copy one word, and force a user-mode page fault...
318 */
319
320.Lcfu_0fupi: subs r2, r2, #4
321 addmi ip, r2, #4
322 bmi .Lcfu_0nowords
323USER( ldrt r3, [r1], #4)
324 str r3, [r0], #4
325 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
326 rsb ip, ip, #0
327 movs ip, ip, lsr #32 - PAGE_SHIFT
328 beq .Lcfu_0fupi
329/*
330 * ip = max no. of bytes to copy before needing another "strt" insn
331 */
332 cmp r2, ip
333 movlt ip, r2
334 sub r2, r2, ip
335 subs ip, ip, #32
336 blt .Lcfu_0rem8lp
337
338.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
339 stmia r0!, {r3 - r6}
340 ldmia r1!, {r3 - r6} @ Shouldnt fault
341 subs ip, ip, #32
342 stmia r0!, {r3 - r6}
343 bpl .Lcfu_0cpy8lp
344
345.Lcfu_0rem8lp: cmn ip, #16
346 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
347 stmgeia r0!, {r3 - r6}
348 tst ip, #8
349 ldmneia r1!, {r3 - r4} @ Shouldnt fault
350 stmneia r0!, {r3 - r4}
351 tst ip, #4
352 ldrnet r3, [r1], #4 @ Shouldnt fault
353 strne r3, [r0], #4
354 ands ip, ip, #3
355 beq .Lcfu_0fupi
356.Lcfu_0nowords: teq ip, #0
357 beq .Lcfu_finished
358.Lcfu_nowords: cmp ip, #2
359USER( ldrbt r3, [r1], #1) @ May fault
360 strb r3, [r0], #1
361USER( ldrgebt r3, [r1], #1) @ May fault
362 strgeb r3, [r0], #1
363USER( ldrgtbt r3, [r1], #1) @ May fault
364 strgtb r3, [r0], #1
365 b .Lcfu_finished
366
367.Lcfu_not_enough:
368 movs ip, r2
369 bne .Lcfu_nowords
370.Lcfu_finished: mov r0, #0
371 add sp, sp, #8
372 LOADREGS(fd,sp!,{r4 - r7, pc})
373
374.Lcfu_src_not_aligned:
375 bic r1, r1, #3
376USER( ldrt r7, [r1], #4) @ May fault
377 cmp ip, #2
378 bgt .Lcfu_3fupi
379 beq .Lcfu_2fupi
380.Lcfu_1fupi: subs r2, r2, #4
381 addmi ip, r2, #4
382 bmi .Lcfu_1nowords
383 mov r3, r7, pull #8
384USER( ldrt r7, [r1], #4) @ May fault
385 orr r3, r3, r7, push #24
386 str r3, [r0], #4
387 mov ip, r1, lsl #32 - PAGE_SHIFT
388 rsb ip, ip, #0
389 movs ip, ip, lsr #32 - PAGE_SHIFT
390 beq .Lcfu_1fupi
391 cmp r2, ip
392 movlt ip, r2
393 sub r2, r2, ip
394 subs ip, ip, #16
395 blt .Lcfu_1rem8lp
396
397.Lcfu_1cpy8lp: mov r3, r7, pull #8
398 ldmia r1!, {r4 - r7} @ Shouldnt fault
399 subs ip, ip, #16
400 orr r3, r3, r4, push #24
401 mov r4, r4, pull #8
402 orr r4, r4, r5, push #24
403 mov r5, r5, pull #8
404 orr r5, r5, r6, push #24
405 mov r6, r6, pull #8
406 orr r6, r6, r7, push #24
407 stmia r0!, {r3 - r6}
408 bpl .Lcfu_1cpy8lp
409
410.Lcfu_1rem8lp: tst ip, #8
411 movne r3, r7, pull #8
412 ldmneia r1!, {r4, r7} @ Shouldnt fault
413 orrne r3, r3, r4, push #24
414 movne r4, r4, pull #8
415 orrne r4, r4, r7, push #24
416 stmneia r0!, {r3 - r4}
417 tst ip, #4
418 movne r3, r7, pull #8
419USER( ldrnet r7, [r1], #4) @ May fault
420 orrne r3, r3, r7, push #24
421 strne r3, [r0], #4
422 ands ip, ip, #3
423 beq .Lcfu_1fupi
424.Lcfu_1nowords: mov r3, r7, get_byte_1
425 teq ip, #0
426 beq .Lcfu_finished
427 cmp ip, #2
428 strb r3, [r0], #1
429 movge r3, r7, get_byte_2
430 strgeb r3, [r0], #1
431 movgt r3, r7, get_byte_3
432 strgtb r3, [r0], #1
433 b .Lcfu_finished
434
435.Lcfu_2fupi: subs r2, r2, #4
436 addmi ip, r2, #4
437 bmi .Lcfu_2nowords
438 mov r3, r7, pull #16
439USER( ldrt r7, [r1], #4) @ May fault
440 orr r3, r3, r7, push #16
441 str r3, [r0], #4
442 mov ip, r1, lsl #32 - PAGE_SHIFT
443 rsb ip, ip, #0
444 movs ip, ip, lsr #32 - PAGE_SHIFT
445 beq .Lcfu_2fupi
446 cmp r2, ip
447 movlt ip, r2
448 sub r2, r2, ip
449 subs ip, ip, #16
450 blt .Lcfu_2rem8lp
451
452
453.Lcfu_2cpy8lp: mov r3, r7, pull #16
454 ldmia r1!, {r4 - r7} @ Shouldnt fault
455 subs ip, ip, #16
456 orr r3, r3, r4, push #16
457 mov r4, r4, pull #16
458 orr r4, r4, r5, push #16
459 mov r5, r5, pull #16
460 orr r5, r5, r6, push #16
461 mov r6, r6, pull #16
462 orr r6, r6, r7, push #16
463 stmia r0!, {r3 - r6}
464 bpl .Lcfu_2cpy8lp
465
466.Lcfu_2rem8lp: tst ip, #8
467 movne r3, r7, pull #16
468 ldmneia r1!, {r4, r7} @ Shouldnt fault
469 orrne r3, r3, r4, push #16
470 movne r4, r4, pull #16
471 orrne r4, r4, r7, push #16
472 stmneia r0!, {r3 - r4}
473 tst ip, #4
474 movne r3, r7, pull #16
475USER( ldrnet r7, [r1], #4) @ May fault
476 orrne r3, r3, r7, push #16
477 strne r3, [r0], #4
478 ands ip, ip, #3
479 beq .Lcfu_2fupi
480.Lcfu_2nowords: mov r3, r7, get_byte_2
481 teq ip, #0
482 beq .Lcfu_finished
483 cmp ip, #2
484 strb r3, [r0], #1
485 movge r3, r7, get_byte_3
486 strgeb r3, [r0], #1
487USER( ldrgtbt r3, [r1], #0) @ May fault
488 strgtb r3, [r0], #1
489 b .Lcfu_finished
490
491.Lcfu_3fupi: subs r2, r2, #4
492 addmi ip, r2, #4
493 bmi .Lcfu_3nowords
494 mov r3, r7, pull #24
495USER( ldrt r7, [r1], #4) @ May fault
496 orr r3, r3, r7, push #8
497 str r3, [r0], #4
498 mov ip, r1, lsl #32 - PAGE_SHIFT
499 rsb ip, ip, #0
500 movs ip, ip, lsr #32 - PAGE_SHIFT
501 beq .Lcfu_3fupi
502 cmp r2, ip
503 movlt ip, r2
504 sub r2, r2, ip
505 subs ip, ip, #16
506 blt .Lcfu_3rem8lp
507
508.Lcfu_3cpy8lp: mov r3, r7, pull #24
509 ldmia r1!, {r4 - r7} @ Shouldnt fault
510 orr r3, r3, r4, push #8
511 mov r4, r4, pull #24
512 orr r4, r4, r5, push #8
513 mov r5, r5, pull #24
514 orr r5, r5, r6, push #8
515 mov r6, r6, pull #24
516 orr r6, r6, r7, push #8
517 stmia r0!, {r3 - r6}
518 subs ip, ip, #16
519 bpl .Lcfu_3cpy8lp
520
521.Lcfu_3rem8lp: tst ip, #8
522 movne r3, r7, pull #24
523 ldmneia r1!, {r4, r7} @ Shouldnt fault
524 orrne r3, r3, r4, push #8
525 movne r4, r4, pull #24
526 orrne r4, r4, r7, push #8
527 stmneia r0!, {r3 - r4}
528 tst ip, #4
529 movne r3, r7, pull #24
530USER( ldrnet r7, [r1], #4) @ May fault
531 orrne r3, r3, r7, push #8
532 strne r3, [r0], #4
533 ands ip, ip, #3
534 beq .Lcfu_3fupi
535.Lcfu_3nowords: mov r3, r7, get_byte_3
536 teq ip, #0
537 beq .Lcfu_finished
538 cmp ip, #2
539 strb r3, [r0], #1
540USER( ldrgebt r3, [r1], #1) @ May fault
541 strgeb r3, [r0], #1
542USER( ldrgtbt r3, [r1], #1) @ May fault
543 strgtb r3, [r0], #1
544 b .Lcfu_finished
545
546 .section .fixup,"ax"
547 .align 0
548 /*
549 * We took an exception. r0 contains a pointer to
550 * the byte not copied.
551 */
5529001: ldr r2, [sp], #4 @ void *to
553 sub r2, r0, r2 @ bytes copied
554 ldr r1, [sp], #4 @ unsigned long count
555 subs r4, r1, r2 @ bytes left to copy
556 movne r1, r4
557 blne __memzero
558 mov r0, r4
559 LOADREGS(fd,sp!, {r4 - r7, pc})
560 .previous
561