Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
[PATCH] ARM: Move copy/clear user_page locking into implementation
Move the locking for copy_user_page() and clear_user_page() into
the implementations which require locking. For simple memcpy/
memset based implementations, the locking is extra overhead which
is not necessary, and prevents preemption occuring.
Signed-off-by: Russell King <rmk@arm.linux.org.uk>
-80
arch/arm/mm/copypage-v4mc.S
-80
arch/arm/mm/copypage-v4mc.S
···
1
-
/*
2
-
* linux/arch/arm/lib/copy_page-armv4mc.S
3
-
*
4
-
* Copyright (C) 1995-2001 Russell King
5
-
*
6
-
* This program is free software; you can redistribute it and/or modify
7
-
* it under the terms of the GNU General Public License version 2 as
8
-
* published by the Free Software Foundation.
9
-
*
10
-
* ASM optimised string functions
11
-
*/
12
-
#include <linux/linkage.h>
13
-
#include <linux/init.h>
14
-
#include <asm/constants.h>
15
-
16
-
.text
17
-
.align 5
18
-
/*
19
-
* ARMv4 mini-dcache optimised copy_user_page
20
-
*
21
-
* We flush the destination cache lines just before we write the data into the
22
-
* corresponding address. Since the Dcache is read-allocate, this removes the
23
-
* Dcache aliasing issue. The writes will be forwarded to the write buffer,
24
-
* and merged as appropriate.
25
-
*
26
-
* Note: We rely on all ARMv4 processors implementing the "invalidate D line"
27
-
* instruction. If your processor does not supply this, you have to write your
28
-
* own copy_user_page that does the right thing.
29
-
*/
30
-
ENTRY(v4_mc_copy_user_page)
31
-
stmfd sp!, {r4, lr} @ 2
32
-
mov r4, r0
33
-
mov r0, r1
34
-
bl map_page_minicache
35
-
mov r1, #PAGE_SZ/64 @ 1
36
-
ldmia r0!, {r2, r3, ip, lr} @ 4
37
-
1: mcr p15, 0, r4, c7, c6, 1 @ 1 invalidate D line
38
-
stmia r4!, {r2, r3, ip, lr} @ 4
39
-
ldmia r0!, {r2, r3, ip, lr} @ 4+1
40
-
stmia r4!, {r2, r3, ip, lr} @ 4
41
-
ldmia r0!, {r2, r3, ip, lr} @ 4
42
-
mcr p15, 0, r4, c7, c6, 1 @ 1 invalidate D line
43
-
stmia r4!, {r2, r3, ip, lr} @ 4
44
-
ldmia r0!, {r2, r3, ip, lr} @ 4
45
-
subs r1, r1, #1 @ 1
46
-
stmia r4!, {r2, r3, ip, lr} @ 4
47
-
ldmneia r0!, {r2, r3, ip, lr} @ 4
48
-
bne 1b @ 1
49
-
ldmfd sp!, {r4, pc} @ 3
50
-
51
-
.align 5
52
-
/*
53
-
* ARMv4 optimised clear_user_page
54
-
*
55
-
* Same story as above.
56
-
*/
57
-
ENTRY(v4_mc_clear_user_page)
58
-
str lr, [sp, #-4]!
59
-
mov r1, #PAGE_SZ/64 @ 1
60
-
mov r2, #0 @ 1
61
-
mov r3, #0 @ 1
62
-
mov ip, #0 @ 1
63
-
mov lr, #0 @ 1
64
-
1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line
65
-
stmia r0!, {r2, r3, ip, lr} @ 4
66
-
stmia r0!, {r2, r3, ip, lr} @ 4
67
-
mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line
68
-
stmia r0!, {r2, r3, ip, lr} @ 4
69
-
stmia r0!, {r2, r3, ip, lr} @ 4
70
-
subs r1, r1, #1 @ 1
71
-
bne 1b @ 1
72
-
ldr pc, [sp], #4
73
-
74
-
__INITDATA
75
-
76
-
.type v4_mc_user_fns, #object
77
-
ENTRY(v4_mc_user_fns)
78
-
.long v4_mc_clear_user_page
79
-
.long v4_mc_copy_user_page
80
-
.size v4_mc_user_fns, . - v4_mc_user_fns
+111
arch/arm/mm/copypage-v4mc.c
+111
arch/arm/mm/copypage-v4mc.c
···
1
+
/*
2
+
* linux/arch/arm/lib/copypage-armv4mc.S
3
+
*
4
+
* Copyright (C) 1995-2005 Russell King
5
+
*
6
+
* This program is free software; you can redistribute it and/or modify
7
+
* it under the terms of the GNU General Public License version 2 as
8
+
* published by the Free Software Foundation.
9
+
*
10
+
* This handles the mini data cache, as found on SA11x0 and XScale
11
+
* processors. When we copy a user page page, we map it in such a way
12
+
* that accesses to this page will not touch the main data cache, but
13
+
* will be cached in the mini data cache. This prevents us thrashing
14
+
* the main data cache on page faults.
15
+
*/
16
+
#include <linux/init.h>
17
+
#include <linux/mm.h>
18
+
19
+
#include <asm/page.h>
20
+
#include <asm/pgtable.h>
21
+
#include <asm/tlbflush.h>
22
+
23
+
/*
24
+
* 0xffff8000 to 0xffffffff is reserved for any ARM architecture
25
+
* specific hacks for copying pages efficiently.
26
+
*/
27
+
#define minicache_pgprot __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | \
28
+
L_PTE_CACHEABLE)
29
+
30
+
#define TOP_PTE(x) pte_offset_kernel(top_pmd, x)
31
+
32
+
static DEFINE_SPINLOCK(minicache_lock);
33
+
34
+
/*
35
+
* ARMv4 mini-dcache optimised copy_user_page
36
+
*
37
+
* We flush the destination cache lines just before we write the data into the
38
+
* corresponding address. Since the Dcache is read-allocate, this removes the
39
+
* Dcache aliasing issue. The writes will be forwarded to the write buffer,
40
+
* and merged as appropriate.
41
+
*
42
+
* Note: We rely on all ARMv4 processors implementing the "invalidate D line"
43
+
* instruction. If your processor does not supply this, you have to write your
44
+
* own copy_user_page that does the right thing.
45
+
*/
46
+
static void __attribute__((naked))
47
+
mc_copy_user_page(void *from, void *to)
48
+
{
49
+
asm volatile(
50
+
"stmfd sp!, {r4, lr} @ 2\n\
51
+
mov r4, %2 @ 1\n\
52
+
ldmia %0!, {r2, r3, ip, lr} @ 4\n\
53
+
1: mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\
54
+
stmia %1!, {r2, r3, ip, lr} @ 4\n\
55
+
ldmia %0!, {r2, r3, ip, lr} @ 4+1\n\
56
+
stmia %1!, {r2, r3, ip, lr} @ 4\n\
57
+
ldmia %0!, {r2, r3, ip, lr} @ 4\n\
58
+
mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\
59
+
stmia %1!, {r2, r3, ip, lr} @ 4\n\
60
+
ldmia %0!, {r2, r3, ip, lr} @ 4\n\
61
+
subs r4, r4, #1 @ 1\n\
62
+
stmia %1!, {r2, r3, ip, lr} @ 4\n\
63
+
ldmneia %0!, {r2, r3, ip, lr} @ 4\n\
64
+
bne 1b @ 1\n\
65
+
ldmfd sp!, {r4, pc} @ 3"
66
+
:
67
+
: "r" (from), "r" (to), "I" (PAGE_SIZE / 64));
68
+
}
69
+
70
+
void v4_mc_copy_user_page(void *kto, const void *kfrom, unsigned long vaddr)
71
+
{
72
+
spin_lock(&minicache_lock);
73
+
74
+
set_pte(TOP_PTE(COPYPAGE_MINICACHE), pfn_pte(__pa(kfrom) >> PAGE_SHIFT, minicache_pgprot));
75
+
flush_tlb_kernel_page(COPYPAGE_MINICACHE);
76
+
77
+
mc_copy_user_page((void *)COPYPAGE_MINICACHE, kto);
78
+
79
+
spin_unlock(&minicache_lock);
80
+
}
81
+
82
+
/*
83
+
* ARMv4 optimised clear_user_page
84
+
*/
85
+
void __attribute__((naked))
86
+
v4_mc_clear_user_page(void *kaddr, unsigned long vaddr)
87
+
{
88
+
asm volatile(
89
+
"str lr, [sp, #-4]!\n\
90
+
mov r1, %0 @ 1\n\
91
+
mov r2, #0 @ 1\n\
92
+
mov r3, #0 @ 1\n\
93
+
mov ip, #0 @ 1\n\
94
+
mov lr, #0 @ 1\n\
95
+
1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
96
+
stmia r0!, {r2, r3, ip, lr} @ 4\n\
97
+
stmia r0!, {r2, r3, ip, lr} @ 4\n\
98
+
mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
99
+
stmia r0!, {r2, r3, ip, lr} @ 4\n\
100
+
stmia r0!, {r2, r3, ip, lr} @ 4\n\
101
+
subs r1, r1, #1 @ 1\n\
102
+
bne 1b @ 1\n\
103
+
ldr pc, [sp], #4"
104
+
:
105
+
: "I" (PAGE_SIZE / 64));
106
+
}
107
+
108
+
struct cpu_user_fns v4_mc_user_fns __initdata = {
109
+
.cpu_clear_user_page = v4_mc_clear_user_page,
110
+
.cpu_copy_user_page = v4_mc_copy_user_page,
111
+
};
+2
-13
include/asm-arm/page.h
+2
-13
include/asm-arm/page.h
···
114
114
unsigned long user);
115
115
#endif
116
116
117
-
#define clear_user_page(addr,vaddr,pg) \
118
-
do { \
119
-
preempt_disable(); \
120
-
__cpu_clear_user_page(addr, vaddr); \
121
-
preempt_enable(); \
122
-
} while (0)
123
-
124
-
#define copy_user_page(to,from,vaddr,pg) \
125
-
do { \
126
-
preempt_disable(); \
127
-
__cpu_copy_user_page(to, from, vaddr); \
128
-
preempt_enable(); \
129
-
} while (0)
117
+
#define clear_user_page(addr,vaddr,pg) __cpu_clear_user_page(addr, vaddr)
118
+
#define copy_user_page(to,from,vaddr,pg) __cpu_copy_user_page(to, from, vaddr)
130
119
131
120
#define clear_page(page) memzero((void *)(page), PAGE_SIZE)
132
121
extern void copy_page(void *to, const void *from);