Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
alpha: fix alignment problem in csum_ipv6_magic()
Hopefully this fixes http://bugzilla.kernel.org/show_bug.cgi?id=8635
The struct in6_addr passed to csum_ipv6_magic() is 4 byte aligned, so we
can't use the regular 64-bit loads. Since the cost of handling of 4 byte
and 1 byte aligned 64-bit data is roughly the same, this code can cope with
any src/dst [mis]alignment.
Signed-off-by: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Dustin Marquess <jailbird@alcatraz.fdf.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
authored by Ivan Kokshaysky and committed by Linus Torvalds 58ed2f9c 653d4876
+38
-15
arch/alpha/lib/csum_ipv6_magic.S
+38
-15
arch/alpha/lib/csum_ipv6_magic.S
···
7
7
* __u32 len,
8
8
* unsigned short proto,
9
9
* unsigned int csum);
10
+
*
11
+
* Misalignment handling (which costs 16 instructions / 8 cycles)
12
+
* added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
10
13
*/
11
14
12
15
.globl csum_ipv6_magic
···
19
16
csum_ipv6_magic:
20
17
.prologue 0
21
18
22
-
ldq $0,0($16) # e0 : load src & dst addr words
19
+
ldq_u $0,0($16) # e0 : load src & dst addr words
23
20
zapnot $20,15,$20 # .. e1 : zero extend incoming csum
24
21
extqh $18,1,$4 # e0 : byte swap len & proto while we wait
25
-
ldq $1,8($16) # .. e1 :
22
+
ldq_u $21,7($16) # .. e1 : handle misalignment
26
23
27
24
extbl $18,1,$5 # e0 :
28
-
ldq $2,0($17) # .. e1 :
25
+
ldq_u $1,8($16) # .. e1 :
29
26
extbl $18,2,$6 # e0 :
30
-
ldq $3,8($17) # .. e1 :
27
+
ldq_u $22,15($16) # .. e1 :
31
28
32
29
extbl $18,3,$18 # e0 :
30
+
ldq_u $2,0($17) # .. e1 :
33
31
sra $4,32,$4 # e0 :
32
+
ldq_u $23,7($17) # .. e1 :
33
+
34
+
extql $0,$16,$0 # e0 :
35
+
ldq_u $3,8($17) # .. e1 :
36
+
extqh $21,$16,$21 # e0 :
37
+
ldq_u $24,15($17) # .. e1 :
38
+
34
39
sll $5,16,$5 # e0 :
40
+
or $0,$21,$0 # .. e1 : 1st src word complete
41
+
extql $1,$16,$1 # e0 :
35
42
addq $20,$0,$20 # .. e1 : begin summing the words
36
43
37
-
sll $6,8,$6 # e0 :
44
+
extqh $22,$16,$22 # e0 :
38
45
cmpult $20,$0,$0 # .. e1 :
39
-
extwh $19,7,$7 # e0 :
46
+
sll $6,8,$6 # e0 :
47
+
or $1,$22,$1 # .. e1 : 2nd src word complete
48
+
49
+
extql $2,$17,$2 # e0 :
40
50
or $4,$18,$18 # .. e1 :
41
-
42
-
extbl $19,1,$19 # e0 :
51
+
extqh $23,$17,$23 # e0 :
43
52
or $5,$6,$5 # .. e1 :
44
-
or $18,$5,$18 # e0 : len complete
45
-
or $19,$7,$19 # .. e1 :
46
53
47
-
sll $19,48,$19 # e0 :
54
+
extql $3,$17,$3 # e0 :
55
+
or $2,$23,$2 # .. e1 : 1st dst word complete
56
+
extqh $24,$17,$24 # e0 :
57
+
or $18,$5,$18 # .. e1 : len complete
58
+
59
+
extwh $19,7,$7 # e0 :
60
+
or $3,$24,$3 # .. e1 : 2nd dst word complete
61
+
extbl $19,1,$19 # e0 :
48
62
addq $20,$1,$20 # .. e1 :
49
-
sra $19,32,$19 # e0 : proto complete
50
-
cmpult $20,$1,$1 # .. e1 :
51
63
52
-
nop # e0 :
64
+
or $19,$7,$19 # e0 :
65
+
cmpult $20,$1,$1 # .. e1 :
66
+
sll $19,48,$19 # e0 :
67
+
nop # .. e0 :
68
+
69
+
sra $19,32,$19 # e0 : proto complete
53
70
addq $20,$2,$20 # .. e1 :
54
71
cmpult $20,$2,$2 # e0 :
55
72
addq $20,$3,$20 # .. e1 :
···
107
84
extwl $0,2,$1 # e0 : fold 17-bit value
108
85
zapnot $0,3,$0 # .. e1 :
109
86
addq $0,$1,$0 # e0 :
110
-
not $0,$0 # e1 : and complement.
87
+
not $0,$0 # .. e1 : and complement.
111
88
112
89
zapnot $0,3,$0 # e0 :
113
90
ret # .. e1 :
+33
-9
arch/alpha/lib/ev6-csum_ipv6_magic.S
+33
-9
arch/alpha/lib/ev6-csum_ipv6_magic.S
···
46
46
* add the 3 low ushorts together, generating a uint
47
47
* a final add of the 2 lower ushorts
48
48
* truncating the result.
49
+
*
50
+
* Misalignment handling added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
51
+
* The cost is 16 instructions (~8 cycles), including two extra loads which
52
+
* may cause additional delay in rare cases (load-load replay traps).
49
53
*/
50
54
51
55
.globl csum_ipv6_magic
···
59
55
csum_ipv6_magic:
60
56
.prologue 0
61
57
62
-
ldq $0,0($16) # L : Latency: 3
58
+
ldq_u $0,0($16) # L : Latency: 3
63
59
inslh $18,7,$4 # U : 0000000000AABBCC
64
-
ldq $1,8($16) # L : Latency: 3
60
+
ldq_u $1,8($16) # L : Latency: 3
65
61
sll $19,8,$7 # U : U L U L : 0x00000000 00aabb00
66
62
63
+
and $16,7,$6 # E : src misalignment
64
+
ldq_u $5,15($16) # L : Latency: 3
67
65
zapnot $20,15,$20 # U : zero extend incoming csum
68
-
ldq $2,0($17) # L : Latency: 3
69
-
sll $19,24,$19 # U : U L L U : 0x000000aa bb000000
66
+
ldq_u $2,0($17) # L : U L U L : Latency: 3
67
+
68
+
extql $0,$6,$0 # U :
69
+
extqh $1,$6,$22 # U :
70
+
ldq_u $3,8($17) # L : Latency: 3
71
+
sll $19,24,$19 # U : U U L U : 0x000000aa bb000000
72
+
73
+
cmoveq $6,$31,$22 # E : src aligned?
74
+
ldq_u $23,15($17) # L : Latency: 3
70
75
inswl $18,3,$18 # U : 000000CCDD000000
76
+
addl $19,$7,$19 # E : U L U L : <sign bits>bbaabb00
71
77
72
-
ldq $3,8($17) # L : Latency: 3
73
-
bis $18,$4,$18 # E : 000000CCDDAABBCC
74
-
addl $19,$7,$19 # E : <sign bits>bbaabb00
75
-
nop # E : U L U L
78
+
or $0,$22,$0 # E : 1st src word complete
79
+
extql $1,$6,$1 # U :
80
+
or $18,$4,$18 # E : 000000CCDDAABBCC
81
+
extqh $5,$6,$5 # U : L U L U
76
82
83
+
and $17,7,$6 # E : dst misalignment
84
+
extql $2,$6,$2 # U :
85
+
or $1,$5,$1 # E : 2nd src word complete
86
+
extqh $3,$6,$22 # U : L U L U :
87
+
88
+
cmoveq $6,$31,$22 # E : dst aligned?
89
+
extql $3,$6,$3 # U :
77
90
addq $20,$0,$20 # E : begin summing the words
91
+
extqh $23,$6,$23 # U : L U L U :
92
+
78
93
srl $18,16,$4 # U : 0000000000CCDDAA
94
+
or $2,$22,$2 # E : 1st dst word complete
79
95
zap $19,0x3,$19 # U : <sign bits>bbaa0000
80
-
nop # E : L U U L
96
+
or $3,$23,$3 # E : U L U L : 2nd dst word complete
81
97
82
98
cmpult $20,$0,$0 # E :
83
99
addq $20,$1,$20 # E :