tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* ----------------------------------------------------------------------- *
2 *
3 * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
8 * Bostom MA 02111-1307, USA; either version 2 of the License, or
9 * (at your option) any later version; incorporated herein by reference.
10 *
11 * ----------------------------------------------------------------------- */
12
13/*
14 * raid6x86.h
15 *
16 * Definitions common to x86 and x86-64 RAID-6 code only
17 */
18
19#ifndef LINUX_RAID_RAID6X86_H
20#define LINUX_RAID_RAID6X86_H
21
22#if defined(__i386__) || defined(__x86_64__)
23
24#ifdef __x86_64__
25
26typedef struct {
27 unsigned int fsave[27];
28 unsigned long cr0;
29} raid6_mmx_save_t __attribute__((aligned(16)));
30
31/* N.B.: For SSE we only save %xmm0-%xmm7 even for x86-64, since
32 the code doesn't know about the additional x86-64 registers */
33typedef struct {
34 unsigned int sarea[8*4+2];
35 unsigned long cr0;
36} raid6_sse_save_t __attribute__((aligned(16)));
37
38/* This is for x86-64-specific code which uses all 16 XMM registers */
39typedef struct {
40 unsigned int sarea[16*4+2];
41 unsigned long cr0;
42} raid6_sse16_save_t __attribute__((aligned(16)));
43
44/* On x86-64 the stack *SHOULD* be 16-byte aligned, but currently this
45 is buggy in the kernel and it's only 8-byte aligned in places, so
46 we need to do this anyway. Sigh. */
47#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
48
49#else /* __i386__ */
50
51typedef struct {
52 unsigned int fsave[27];
53 unsigned long cr0;
54} raid6_mmx_save_t;
55
56/* On i386, the stack is only 8-byte aligned, but SSE requires 16-byte
57 alignment. The +3 is so we have the slack space to manually align
58 a properly-sized area correctly. */
59typedef struct {
60 unsigned int sarea[8*4+3];
61 unsigned long cr0;
62} raid6_sse_save_t;
63
64/* Find the 16-byte aligned save area */
65#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
66
67#endif
68
69#ifdef __KERNEL__ /* Real code */
70
71/* Note: %cr0 is 32 bits on i386 and 64 bits on x86-64 */
72
73static inline unsigned long raid6_get_fpu(void)
74{
75 unsigned long cr0;
76
77 preempt_disable();
78 asm volatile("mov %%cr0,%0 ; clts" : "=r" (cr0));
79 return cr0;
80}
81
82static inline void raid6_put_fpu(unsigned long cr0)
83{
84 asm volatile("mov %0,%%cr0" : : "r" (cr0));
85 preempt_enable();
86}
87
88#else /* Dummy code for user space testing */
89
90static inline unsigned long raid6_get_fpu(void)
91{
92 return 0xf00ba6;
93}
94
95static inline void raid6_put_fpu(unsigned long cr0)
96{
97 (void)cr0;
98}
99
100#endif
101
102static inline void raid6_before_mmx(raid6_mmx_save_t *s)
103{
104 s->cr0 = raid6_get_fpu();
105 asm volatile("fsave %0 ; fwait" : "=m" (s->fsave[0]));
106}
107
108static inline void raid6_after_mmx(raid6_mmx_save_t *s)
109{
110 asm volatile("frstor %0" : : "m" (s->fsave[0]));
111 raid6_put_fpu(s->cr0);
112}
113
114static inline void raid6_before_sse(raid6_sse_save_t *s)
115{
116 unsigned int *rsa = SAREA(s);
117
118 s->cr0 = raid6_get_fpu();
119
120 asm volatile("movaps %%xmm0,%0" : "=m" (rsa[0]));
121 asm volatile("movaps %%xmm1,%0" : "=m" (rsa[4]));
122 asm volatile("movaps %%xmm2,%0" : "=m" (rsa[8]));
123 asm volatile("movaps %%xmm3,%0" : "=m" (rsa[12]));
124 asm volatile("movaps %%xmm4,%0" : "=m" (rsa[16]));
125 asm volatile("movaps %%xmm5,%0" : "=m" (rsa[20]));
126 asm volatile("movaps %%xmm6,%0" : "=m" (rsa[24]));
127 asm volatile("movaps %%xmm7,%0" : "=m" (rsa[28]));
128}
129
130static inline void raid6_after_sse(raid6_sse_save_t *s)
131{
132 unsigned int *rsa = SAREA(s);
133
134 asm volatile("movaps %0,%%xmm0" : : "m" (rsa[0]));
135 asm volatile("movaps %0,%%xmm1" : : "m" (rsa[4]));
136 asm volatile("movaps %0,%%xmm2" : : "m" (rsa[8]));
137 asm volatile("movaps %0,%%xmm3" : : "m" (rsa[12]));
138 asm volatile("movaps %0,%%xmm4" : : "m" (rsa[16]));
139 asm volatile("movaps %0,%%xmm5" : : "m" (rsa[20]));
140 asm volatile("movaps %0,%%xmm6" : : "m" (rsa[24]));
141 asm volatile("movaps %0,%%xmm7" : : "m" (rsa[28]));
142
143 raid6_put_fpu(s->cr0);
144}
145
146static inline void raid6_before_sse2(raid6_sse_save_t *s)
147{
148 unsigned int *rsa = SAREA(s);
149
150 s->cr0 = raid6_get_fpu();
151
152 asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
153 asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
154 asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
155 asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
156 asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
157 asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
158 asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
159 asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
160}
161
162static inline void raid6_after_sse2(raid6_sse_save_t *s)
163{
164 unsigned int *rsa = SAREA(s);
165
166 asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
167 asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
168 asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
169 asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
170 asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
171 asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
172 asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
173 asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
174
175 raid6_put_fpu(s->cr0);
176}
177
178#ifdef __x86_64__
179
180static inline void raid6_before_sse16(raid6_sse16_save_t *s)
181{
182 unsigned int *rsa = SAREA(s);
183
184 s->cr0 = raid6_get_fpu();
185
186 asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
187 asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
188 asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
189 asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
190 asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
191 asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
192 asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
193 asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
194 asm volatile("movdqa %%xmm8,%0" : "=m" (rsa[32]));
195 asm volatile("movdqa %%xmm9,%0" : "=m" (rsa[36]));
196 asm volatile("movdqa %%xmm10,%0" : "=m" (rsa[40]));
197 asm volatile("movdqa %%xmm11,%0" : "=m" (rsa[44]));
198 asm volatile("movdqa %%xmm12,%0" : "=m" (rsa[48]));
199 asm volatile("movdqa %%xmm13,%0" : "=m" (rsa[52]));
200 asm volatile("movdqa %%xmm14,%0" : "=m" (rsa[56]));
201 asm volatile("movdqa %%xmm15,%0" : "=m" (rsa[60]));
202}
203
204static inline void raid6_after_sse16(raid6_sse16_save_t *s)
205{
206 unsigned int *rsa = SAREA(s);
207
208 asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
209 asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
210 asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
211 asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
212 asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
213 asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
214 asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
215 asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
216 asm volatile("movdqa %0,%%xmm8" : : "m" (rsa[32]));
217 asm volatile("movdqa %0,%%xmm9" : : "m" (rsa[36]));
218 asm volatile("movdqa %0,%%xmm10" : : "m" (rsa[40]));
219 asm volatile("movdqa %0,%%xmm11" : : "m" (rsa[44]));
220 asm volatile("movdqa %0,%%xmm12" : : "m" (rsa[48]));
221 asm volatile("movdqa %0,%%xmm13" : : "m" (rsa[52]));
222 asm volatile("movdqa %0,%%xmm14" : : "m" (rsa[56]));
223 asm volatile("movdqa %0,%%xmm15" : : "m" (rsa[60]));
224
225 raid6_put_fpu(s->cr0);
226}
227
228#endif /* __x86_64__ */
229
230/* User space test hack */
231#ifndef __KERNEL__
232static inline int cpuid_features(void)
233{
234 u32 eax = 1;
235 u32 ebx, ecx, edx;
236
237 asm volatile("cpuid" :
238 "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx));
239
240 return edx;
241}
242#endif /* ndef __KERNEL__ */
243
244#endif
245#endif