Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
sparc64: Make rwsems 64-bit.
Basically tip-off the powerpc code, use a 64-bit type and atomic64_t
interfaces for the implementation.
This gets us off of the by-hand asm code I wrote, which frankly I
think probably ruins I-cache hit rates.
The idea was the keep the call chains less deep, but anything taking
the rw-semaphores probably is also calling other stuff and therefore
already has allocated a stack-frame. So no real stack frame savings
ever.
Ben H. has posted patches to make powerpc use 64-bit too and with some
abstractions we can probably use a shared header file somewhere.
With suggestions from Sam Ravnborg.
Signed-off-by: David S. Miller <davem@davemloft.net>
-12
arch/sparc/include/asm/rwsem-const.h
-12
arch/sparc/include/asm/rwsem-const.h
···
1
-
/* rwsem-const.h: RW semaphore counter constants. */
2
-
#ifndef _SPARC64_RWSEM_CONST_H
3
-
#define _SPARC64_RWSEM_CONST_H
4
-
5
-
#define RWSEM_UNLOCKED_VALUE 0x00000000
6
-
#define RWSEM_ACTIVE_BIAS 0x00000001
7
-
#define RWSEM_ACTIVE_MASK 0x0000ffff
8
-
#define RWSEM_WAITING_BIAS (-0x00010000)
9
-
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
10
-
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
11
-
12
-
#endif /* _SPARC64_RWSEM_CONST_H */
+103
-17
arch/sparc/include/asm/rwsem.h
+103
-17
arch/sparc/include/asm/rwsem.h
···
15
15
16
16
#include <linux/list.h>
17
17
#include <linux/spinlock.h>
18
-
#include <asm/rwsem-const.h>
19
18
20
19
struct rwsem_waiter;
21
20
22
21
struct rw_semaphore {
23
-
signed int count;
24
-
spinlock_t wait_lock;
25
-
struct list_head wait_list;
22
+
signed long count;
23
+
#define RWSEM_UNLOCKED_VALUE 0x00000000L
24
+
#define RWSEM_ACTIVE_BIAS 0x00000001L
25
+
#define RWSEM_ACTIVE_MASK 0xffffffffL
26
+
#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
27
+
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
28
+
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
29
+
spinlock_t wait_lock;
30
+
struct list_head wait_list;
26
31
#ifdef CONFIG_DEBUG_LOCK_ALLOC
27
-
struct lockdep_map dep_map;
32
+
struct lockdep_map dep_map;
28
33
#endif
29
34
};
30
35
···
46
41
#define DECLARE_RWSEM(name) \
47
42
struct rw_semaphore name = __RWSEM_INITIALIZER(name)
48
43
44
+
extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
45
+
extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
46
+
extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);
47
+
extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
48
+
49
49
extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
50
50
struct lock_class_key *key);
51
51
···
61
51
__init_rwsem((sem), #sem, &__key); \
62
52
} while (0)
63
53
64
-
extern void __down_read(struct rw_semaphore *sem);
65
-
extern int __down_read_trylock(struct rw_semaphore *sem);
66
-
extern void __down_write(struct rw_semaphore *sem);
67
-
extern int __down_write_trylock(struct rw_semaphore *sem);
68
-
extern void __up_read(struct rw_semaphore *sem);
69
-
extern void __up_write(struct rw_semaphore *sem);
70
-
extern void __downgrade_write(struct rw_semaphore *sem);
54
+
/*
55
+
* lock for reading
56
+
*/
57
+
static inline void __down_read(struct rw_semaphore *sem)
58
+
{
59
+
if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
60
+
rwsem_down_read_failed(sem);
61
+
}
71
62
63
+
static inline int __down_read_trylock(struct rw_semaphore *sem)
64
+
{
65
+
long tmp;
66
+
67
+
while ((tmp = sem->count) >= 0L) {
68
+
if (tmp == cmpxchg(&sem->count, tmp,
69
+
tmp + RWSEM_ACTIVE_READ_BIAS)) {
70
+
return 1;
71
+
}
72
+
}
73
+
return 0;
74
+
}
75
+
76
+
/*
77
+
* lock for writing
78
+
*/
72
79
static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
73
80
{
74
-
__down_write(sem);
81
+
long tmp;
82
+
83
+
tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
84
+
(atomic64_t *)(&sem->count));
85
+
if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
86
+
rwsem_down_write_failed(sem);
75
87
}
76
88
77
-
static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
89
+
static inline void __down_write(struct rw_semaphore *sem)
78
90
{
79
-
return atomic_add_return(delta, (atomic_t *)(&sem->count));
91
+
__down_write_nested(sem, 0);
80
92
}
81
93
82
-
static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
94
+
static inline int __down_write_trylock(struct rw_semaphore *sem)
83
95
{
84
-
atomic_add(delta, (atomic_t *)(&sem->count));
96
+
long tmp;
97
+
98
+
tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
99
+
RWSEM_ACTIVE_WRITE_BIAS);
100
+
return tmp == RWSEM_UNLOCKED_VALUE;
101
+
}
102
+
103
+
/*
104
+
* unlock after reading
105
+
*/
106
+
static inline void __up_read(struct rw_semaphore *sem)
107
+
{
108
+
long tmp;
109
+
110
+
tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
111
+
if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
112
+
rwsem_wake(sem);
113
+
}
114
+
115
+
/*
116
+
* unlock after writing
117
+
*/
118
+
static inline void __up_write(struct rw_semaphore *sem)
119
+
{
120
+
if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
121
+
(atomic64_t *)(&sem->count)) < 0L))
122
+
rwsem_wake(sem);
123
+
}
124
+
125
+
/*
126
+
* implement atomic add functionality
127
+
*/
128
+
static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
129
+
{
130
+
atomic64_add(delta, (atomic64_t *)(&sem->count));
131
+
}
132
+
133
+
/*
134
+
* downgrade write lock to read lock
135
+
*/
136
+
static inline void __downgrade_write(struct rw_semaphore *sem)
137
+
{
138
+
long tmp;
139
+
140
+
tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
141
+
if (tmp < 0L)
142
+
rwsem_downgrade_wake(sem);
143
+
}
144
+
145
+
/*
146
+
* implement exchange and add functionality
147
+
*/
148
+
static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
149
+
{
150
+
return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
85
151
}
86
152
87
153
static inline int rwsem_is_locked(struct rw_semaphore *sem)
+1
-1
arch/sparc/lib/Makefile
+1
-1
arch/sparc/lib/Makefile
···
15
15
lib-$(CONFIG_SPARC32) += copy_user.o locks.o
16
16
lib-y += atomic_$(BITS).o
17
17
lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o
18
-
lib-y += rwsem_$(BITS).o
18
+
lib-$(CONFIG_SPARC32) += rwsem_32.o
19
19
lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o
20
20
21
21
lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o
-163
arch/sparc/lib/rwsem_64.S
-163
arch/sparc/lib/rwsem_64.S
···
1
-
/* rwsem.S: RW semaphore assembler.
2
-
*
3
-
* Written by David S. Miller (davem@redhat.com), 2001.
4
-
* Derived from asm-i386/rwsem.h
5
-
*/
6
-
7
-
#include <asm/rwsem-const.h>
8
-
9
-
.section .sched.text, "ax"
10
-
11
-
.globl __down_read
12
-
__down_read:
13
-
1: lduw [%o0], %g1
14
-
add %g1, 1, %g7
15
-
cas [%o0], %g1, %g7
16
-
cmp %g1, %g7
17
-
bne,pn %icc, 1b
18
-
add %g7, 1, %g7
19
-
cmp %g7, 0
20
-
bl,pn %icc, 3f
21
-
nop
22
-
2:
23
-
retl
24
-
nop
25
-
3:
26
-
save %sp, -192, %sp
27
-
call rwsem_down_read_failed
28
-
mov %i0, %o0
29
-
ret
30
-
restore
31
-
.size __down_read, .-__down_read
32
-
33
-
.globl __down_read_trylock
34
-
__down_read_trylock:
35
-
1: lduw [%o0], %g1
36
-
add %g1, 1, %g7
37
-
cmp %g7, 0
38
-
bl,pn %icc, 2f
39
-
mov 0, %o1
40
-
cas [%o0], %g1, %g7
41
-
cmp %g1, %g7
42
-
bne,pn %icc, 1b
43
-
mov 1, %o1
44
-
2: retl
45
-
mov %o1, %o0
46
-
.size __down_read_trylock, .-__down_read_trylock
47
-
48
-
.globl __down_write
49
-
__down_write:
50
-
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
51
-
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
52
-
1:
53
-
lduw [%o0], %g3
54
-
add %g3, %g1, %g7
55
-
cas [%o0], %g3, %g7
56
-
cmp %g3, %g7
57
-
bne,pn %icc, 1b
58
-
cmp %g7, 0
59
-
bne,pn %icc, 3f
60
-
nop
61
-
2: retl
62
-
nop
63
-
3:
64
-
save %sp, -192, %sp
65
-
call rwsem_down_write_failed
66
-
mov %i0, %o0
67
-
ret
68
-
restore
69
-
.size __down_write, .-__down_write
70
-
71
-
.globl __down_write_trylock
72
-
__down_write_trylock:
73
-
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
74
-
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
75
-
1:
76
-
lduw [%o0], %g3
77
-
cmp %g3, 0
78
-
bne,pn %icc, 2f
79
-
mov 0, %o1
80
-
add %g3, %g1, %g7
81
-
cas [%o0], %g3, %g7
82
-
cmp %g3, %g7
83
-
bne,pn %icc, 1b
84
-
mov 1, %o1
85
-
2: retl
86
-
mov %o1, %o0
87
-
.size __down_write_trylock, .-__down_write_trylock
88
-
89
-
.globl __up_read
90
-
__up_read:
91
-
1:
92
-
lduw [%o0], %g1
93
-
sub %g1, 1, %g7
94
-
cas [%o0], %g1, %g7
95
-
cmp %g1, %g7
96
-
bne,pn %icc, 1b
97
-
cmp %g7, 0
98
-
bl,pn %icc, 3f
99
-
nop
100
-
2: retl
101
-
nop
102
-
3: sethi %hi(RWSEM_ACTIVE_MASK), %g1
103
-
sub %g7, 1, %g7
104
-
or %g1, %lo(RWSEM_ACTIVE_MASK), %g1
105
-
andcc %g7, %g1, %g0
106
-
bne,pn %icc, 2b
107
-
nop
108
-
save %sp, -192, %sp
109
-
call rwsem_wake
110
-
mov %i0, %o0
111
-
ret
112
-
restore
113
-
.size __up_read, .-__up_read
114
-
115
-
.globl __up_write
116
-
__up_write:
117
-
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
118
-
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
119
-
1:
120
-
lduw [%o0], %g3
121
-
sub %g3, %g1, %g7
122
-
cas [%o0], %g3, %g7
123
-
cmp %g3, %g7
124
-
bne,pn %icc, 1b
125
-
sub %g7, %g1, %g7
126
-
cmp %g7, 0
127
-
bl,pn %icc, 3f
128
-
nop
129
-
2:
130
-
retl
131
-
nop
132
-
3:
133
-
save %sp, -192, %sp
134
-
call rwsem_wake
135
-
mov %i0, %o0
136
-
ret
137
-
restore
138
-
.size __up_write, .-__up_write
139
-
140
-
.globl __downgrade_write
141
-
__downgrade_write:
142
-
sethi %hi(RWSEM_WAITING_BIAS), %g1
143
-
or %g1, %lo(RWSEM_WAITING_BIAS), %g1
144
-
1:
145
-
lduw [%o0], %g3
146
-
sub %g3, %g1, %g7
147
-
cas [%o0], %g3, %g7
148
-
cmp %g3, %g7
149
-
bne,pn %icc, 1b
150
-
sub %g7, %g1, %g7
151
-
cmp %g7, 0
152
-
bl,pn %icc, 3f
153
-
nop
154
-
2:
155
-
retl
156
-
nop
157
-
3:
158
-
save %sp, -192, %sp
159
-
call rwsem_downgrade_wake
160
-
mov %i0, %o0
161
-
ret
162
-
restore
163
-
.size __downgrade_write, .-__downgrade_write