tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * RAID-6 syndrome calculation using RISC-V vector instructions
4 *
5 * Copyright 2024 Institute of Software, CAS.
6 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
7 *
8 * Based on neon.uc:
9 * Copyright 2002-2004 H. Peter Anvin
10 */
11
12#include "rvv.h"
13
14#ifdef __riscv_vector
15#error "This code must be built without compiler support for vector"
16#endif
17
18static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
19{
20 u8 **dptr = (u8 **)ptrs;
21 u8 *p, *q;
22 unsigned long vl, d, nsize;
23 int z, z0;
24
25 z0 = disks - 3; /* Highest data disk */
26 p = dptr[z0 + 1]; /* XOR parity */
27 q = dptr[z0 + 2]; /* RS syndrome */
28
29 asm volatile (".option push\n"
30 ".option arch,+v\n"
31 "vsetvli %0, x0, e8, m1, ta, ma\n"
32 ".option pop\n"
33 : "=&r" (vl)
34 );
35
36 nsize = vl;
37
38 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
39 for (d = 0; d < bytes; d += nsize * 1) {
40 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
41 asm volatile (".option push\n"
42 ".option arch,+v\n"
43 "vle8.v v0, (%[wp0])\n"
44 "vmv.v.v v1, v0\n"
45 ".option pop\n"
46 : :
47 [wp0]"r"(&dptr[z0][d + 0 * nsize])
48 );
49
50 for (z = z0 - 1 ; z >= 0 ; z--) {
51 /*
52 * w2$$ = MASK(wq$$);
53 * w1$$ = SHLBYTE(wq$$);
54 * w2$$ &= NBYTES(0x1d);
55 * w1$$ ^= w2$$;
56 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
57 * wq$$ = w1$$ ^ wd$$;
58 * wp$$ ^= wd$$;
59 */
60 asm volatile (".option push\n"
61 ".option arch,+v\n"
62 "vsra.vi v2, v1, 7\n"
63 "vsll.vi v3, v1, 1\n"
64 "vand.vx v2, v2, %[x1d]\n"
65 "vxor.vv v3, v3, v2\n"
66 "vle8.v v2, (%[wd0])\n"
67 "vxor.vv v1, v3, v2\n"
68 "vxor.vv v0, v0, v2\n"
69 ".option pop\n"
70 : :
71 [wd0]"r"(&dptr[z][d + 0 * nsize]),
72 [x1d]"r"(0x1d)
73 );
74 }
75
76 /*
77 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
78 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
79 */
80 asm volatile (".option push\n"
81 ".option arch,+v\n"
82 "vse8.v v0, (%[wp0])\n"
83 "vse8.v v1, (%[wq0])\n"
84 ".option pop\n"
85 : :
86 [wp0]"r"(&p[d + nsize * 0]),
87 [wq0]"r"(&q[d + nsize * 0])
88 );
89 }
90}
91
92static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
93 unsigned long bytes, void **ptrs)
94{
95 u8 **dptr = (u8 **)ptrs;
96 u8 *p, *q;
97 unsigned long vl, d, nsize;
98 int z, z0;
99
100 z0 = stop; /* P/Q right side optimization */
101 p = dptr[disks - 2]; /* XOR parity */
102 q = dptr[disks - 1]; /* RS syndrome */
103
104 asm volatile (".option push\n"
105 ".option arch,+v\n"
106 "vsetvli %0, x0, e8, m1, ta, ma\n"
107 ".option pop\n"
108 : "=&r" (vl)
109 );
110
111 nsize = vl;
112
113 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
114 for (d = 0 ; d < bytes ; d += nsize * 1) {
115 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
116 asm volatile (".option push\n"
117 ".option arch,+v\n"
118 "vle8.v v0, (%[wp0])\n"
119 "vmv.v.v v1, v0\n"
120 ".option pop\n"
121 : :
122 [wp0]"r"(&dptr[z0][d + 0 * nsize])
123 );
124
125 /* P/Q data pages */
126 for (z = z0 - 1; z >= start; z--) {
127 /*
128 * w2$$ = MASK(wq$$);
129 * w1$$ = SHLBYTE(wq$$);
130 * w2$$ &= NBYTES(0x1d);
131 * w1$$ ^= w2$$;
132 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
133 * wq$$ = w1$$ ^ wd$$;
134 * wp$$ ^= wd$$;
135 */
136 asm volatile (".option push\n"
137 ".option arch,+v\n"
138 "vsra.vi v2, v1, 7\n"
139 "vsll.vi v3, v1, 1\n"
140 "vand.vx v2, v2, %[x1d]\n"
141 "vxor.vv v3, v3, v2\n"
142 "vle8.v v2, (%[wd0])\n"
143 "vxor.vv v1, v3, v2\n"
144 "vxor.vv v0, v0, v2\n"
145 ".option pop\n"
146 : :
147 [wd0]"r"(&dptr[z][d + 0 * nsize]),
148 [x1d]"r"(0x1d)
149 );
150 }
151
152 /* P/Q left side optimization */
153 for (z = start - 1; z >= 0; z--) {
154 /*
155 * w2$$ = MASK(wq$$);
156 * w1$$ = SHLBYTE(wq$$);
157 * w2$$ &= NBYTES(0x1d);
158 * wq$$ = w1$$ ^ w2$$;
159 */
160 asm volatile (".option push\n"
161 ".option arch,+v\n"
162 "vsra.vi v2, v1, 7\n"
163 "vsll.vi v3, v1, 1\n"
164 "vand.vx v2, v2, %[x1d]\n"
165 "vxor.vv v1, v3, v2\n"
166 ".option pop\n"
167 : :
168 [x1d]"r"(0x1d)
169 );
170 }
171
172 /*
173 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
174 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
175 * v0:wp0, v1:wq0, v2:p0, v3:q0
176 */
177 asm volatile (".option push\n"
178 ".option arch,+v\n"
179 "vle8.v v2, (%[wp0])\n"
180 "vle8.v v3, (%[wq0])\n"
181 "vxor.vv v2, v2, v0\n"
182 "vxor.vv v3, v3, v1\n"
183 "vse8.v v2, (%[wp0])\n"
184 "vse8.v v3, (%[wq0])\n"
185 ".option pop\n"
186 : :
187 [wp0]"r"(&p[d + nsize * 0]),
188 [wq0]"r"(&q[d + nsize * 0])
189 );
190 }
191}
192
193static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
194{
195 u8 **dptr = (u8 **)ptrs;
196 u8 *p, *q;
197 unsigned long vl, d, nsize;
198 int z, z0;
199
200 z0 = disks - 3; /* Highest data disk */
201 p = dptr[z0 + 1]; /* XOR parity */
202 q = dptr[z0 + 2]; /* RS syndrome */
203
204 asm volatile (".option push\n"
205 ".option arch,+v\n"
206 "vsetvli %0, x0, e8, m1, ta, ma\n"
207 ".option pop\n"
208 : "=&r" (vl)
209 );
210
211 nsize = vl;
212
213 /*
214 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
215 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
216 */
217 for (d = 0; d < bytes; d += nsize * 2) {
218 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
219 asm volatile (".option push\n"
220 ".option arch,+v\n"
221 "vle8.v v0, (%[wp0])\n"
222 "vmv.v.v v1, v0\n"
223 "vle8.v v4, (%[wp1])\n"
224 "vmv.v.v v5, v4\n"
225 ".option pop\n"
226 : :
227 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
228 [wp1]"r"(&dptr[z0][d + 1 * nsize])
229 );
230
231 for (z = z0 - 1; z >= 0; z--) {
232 /*
233 * w2$$ = MASK(wq$$);
234 * w1$$ = SHLBYTE(wq$$);
235 * w2$$ &= NBYTES(0x1d);
236 * w1$$ ^= w2$$;
237 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
238 * wq$$ = w1$$ ^ wd$$;
239 * wp$$ ^= wd$$;
240 */
241 asm volatile (".option push\n"
242 ".option arch,+v\n"
243 "vsra.vi v2, v1, 7\n"
244 "vsll.vi v3, v1, 1\n"
245 "vand.vx v2, v2, %[x1d]\n"
246 "vxor.vv v3, v3, v2\n"
247 "vle8.v v2, (%[wd0])\n"
248 "vxor.vv v1, v3, v2\n"
249 "vxor.vv v0, v0, v2\n"
250
251 "vsra.vi v6, v5, 7\n"
252 "vsll.vi v7, v5, 1\n"
253 "vand.vx v6, v6, %[x1d]\n"
254 "vxor.vv v7, v7, v6\n"
255 "vle8.v v6, (%[wd1])\n"
256 "vxor.vv v5, v7, v6\n"
257 "vxor.vv v4, v4, v6\n"
258 ".option pop\n"
259 : :
260 [wd0]"r"(&dptr[z][d + 0 * nsize]),
261 [wd1]"r"(&dptr[z][d + 1 * nsize]),
262 [x1d]"r"(0x1d)
263 );
264 }
265
266 /*
267 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
268 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
269 */
270 asm volatile (".option push\n"
271 ".option arch,+v\n"
272 "vse8.v v0, (%[wp0])\n"
273 "vse8.v v1, (%[wq0])\n"
274 "vse8.v v4, (%[wp1])\n"
275 "vse8.v v5, (%[wq1])\n"
276 ".option pop\n"
277 : :
278 [wp0]"r"(&p[d + nsize * 0]),
279 [wq0]"r"(&q[d + nsize * 0]),
280 [wp1]"r"(&p[d + nsize * 1]),
281 [wq1]"r"(&q[d + nsize * 1])
282 );
283 }
284}
285
286static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
287 unsigned long bytes, void **ptrs)
288{
289 u8 **dptr = (u8 **)ptrs;
290 u8 *p, *q;
291 unsigned long vl, d, nsize;
292 int z, z0;
293
294 z0 = stop; /* P/Q right side optimization */
295 p = dptr[disks - 2]; /* XOR parity */
296 q = dptr[disks - 1]; /* RS syndrome */
297
298 asm volatile (".option push\n"
299 ".option arch,+v\n"
300 "vsetvli %0, x0, e8, m1, ta, ma\n"
301 ".option pop\n"
302 : "=&r" (vl)
303 );
304
305 nsize = vl;
306
307 /*
308 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
309 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
310 */
311 for (d = 0; d < bytes; d += nsize * 2) {
312 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
313 asm volatile (".option push\n"
314 ".option arch,+v\n"
315 "vle8.v v0, (%[wp0])\n"
316 "vmv.v.v v1, v0\n"
317 "vle8.v v4, (%[wp1])\n"
318 "vmv.v.v v5, v4\n"
319 ".option pop\n"
320 : :
321 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
322 [wp1]"r"(&dptr[z0][d + 1 * nsize])
323 );
324
325 /* P/Q data pages */
326 for (z = z0 - 1; z >= start; z--) {
327 /*
328 * w2$$ = MASK(wq$$);
329 * w1$$ = SHLBYTE(wq$$);
330 * w2$$ &= NBYTES(0x1d);
331 * w1$$ ^= w2$$;
332 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
333 * wq$$ = w1$$ ^ wd$$;
334 * wp$$ ^= wd$$;
335 */
336 asm volatile (".option push\n"
337 ".option arch,+v\n"
338 "vsra.vi v2, v1, 7\n"
339 "vsll.vi v3, v1, 1\n"
340 "vand.vx v2, v2, %[x1d]\n"
341 "vxor.vv v3, v3, v2\n"
342 "vle8.v v2, (%[wd0])\n"
343 "vxor.vv v1, v3, v2\n"
344 "vxor.vv v0, v0, v2\n"
345
346 "vsra.vi v6, v5, 7\n"
347 "vsll.vi v7, v5, 1\n"
348 "vand.vx v6, v6, %[x1d]\n"
349 "vxor.vv v7, v7, v6\n"
350 "vle8.v v6, (%[wd1])\n"
351 "vxor.vv v5, v7, v6\n"
352 "vxor.vv v4, v4, v6\n"
353 ".option pop\n"
354 : :
355 [wd0]"r"(&dptr[z][d + 0 * nsize]),
356 [wd1]"r"(&dptr[z][d + 1 * nsize]),
357 [x1d]"r"(0x1d)
358 );
359 }
360
361 /* P/Q left side optimization */
362 for (z = start - 1; z >= 0; z--) {
363 /*
364 * w2$$ = MASK(wq$$);
365 * w1$$ = SHLBYTE(wq$$);
366 * w2$$ &= NBYTES(0x1d);
367 * wq$$ = w1$$ ^ w2$$;
368 */
369 asm volatile (".option push\n"
370 ".option arch,+v\n"
371 "vsra.vi v2, v1, 7\n"
372 "vsll.vi v3, v1, 1\n"
373 "vand.vx v2, v2, %[x1d]\n"
374 "vxor.vv v1, v3, v2\n"
375
376 "vsra.vi v6, v5, 7\n"
377 "vsll.vi v7, v5, 1\n"
378 "vand.vx v6, v6, %[x1d]\n"
379 "vxor.vv v5, v7, v6\n"
380 ".option pop\n"
381 : :
382 [x1d]"r"(0x1d)
383 );
384 }
385
386 /*
387 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
388 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
389 * v0:wp0, v1:wq0, v2:p0, v3:q0
390 * v4:wp1, v5:wq1, v6:p1, v7:q1
391 */
392 asm volatile (".option push\n"
393 ".option arch,+v\n"
394 "vle8.v v2, (%[wp0])\n"
395 "vle8.v v3, (%[wq0])\n"
396 "vxor.vv v2, v2, v0\n"
397 "vxor.vv v3, v3, v1\n"
398 "vse8.v v2, (%[wp0])\n"
399 "vse8.v v3, (%[wq0])\n"
400
401 "vle8.v v6, (%[wp1])\n"
402 "vle8.v v7, (%[wq1])\n"
403 "vxor.vv v6, v6, v4\n"
404 "vxor.vv v7, v7, v5\n"
405 "vse8.v v6, (%[wp1])\n"
406 "vse8.v v7, (%[wq1])\n"
407 ".option pop\n"
408 : :
409 [wp0]"r"(&p[d + nsize * 0]),
410 [wq0]"r"(&q[d + nsize * 0]),
411 [wp1]"r"(&p[d + nsize * 1]),
412 [wq1]"r"(&q[d + nsize * 1])
413 );
414 }
415}
416
417static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
418{
419 u8 **dptr = (u8 **)ptrs;
420 u8 *p, *q;
421 unsigned long vl, d, nsize;
422 int z, z0;
423
424 z0 = disks - 3; /* Highest data disk */
425 p = dptr[z0 + 1]; /* XOR parity */
426 q = dptr[z0 + 2]; /* RS syndrome */
427
428 asm volatile (".option push\n"
429 ".option arch,+v\n"
430 "vsetvli %0, x0, e8, m1, ta, ma\n"
431 ".option pop\n"
432 : "=&r" (vl)
433 );
434
435 nsize = vl;
436
437 /*
438 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
439 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
440 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
441 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
442 */
443 for (d = 0; d < bytes; d += nsize * 4) {
444 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
445 asm volatile (".option push\n"
446 ".option arch,+v\n"
447 "vle8.v v0, (%[wp0])\n"
448 "vmv.v.v v1, v0\n"
449 "vle8.v v4, (%[wp1])\n"
450 "vmv.v.v v5, v4\n"
451 "vle8.v v8, (%[wp2])\n"
452 "vmv.v.v v9, v8\n"
453 "vle8.v v12, (%[wp3])\n"
454 "vmv.v.v v13, v12\n"
455 ".option pop\n"
456 : :
457 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
458 [wp1]"r"(&dptr[z0][d + 1 * nsize]),
459 [wp2]"r"(&dptr[z0][d + 2 * nsize]),
460 [wp3]"r"(&dptr[z0][d + 3 * nsize])
461 );
462
463 for (z = z0 - 1; z >= 0; z--) {
464 /*
465 * w2$$ = MASK(wq$$);
466 * w1$$ = SHLBYTE(wq$$);
467 * w2$$ &= NBYTES(0x1d);
468 * w1$$ ^= w2$$;
469 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
470 * wq$$ = w1$$ ^ wd$$;
471 * wp$$ ^= wd$$;
472 */
473 asm volatile (".option push\n"
474 ".option arch,+v\n"
475 "vsra.vi v2, v1, 7\n"
476 "vsll.vi v3, v1, 1\n"
477 "vand.vx v2, v2, %[x1d]\n"
478 "vxor.vv v3, v3, v2\n"
479 "vle8.v v2, (%[wd0])\n"
480 "vxor.vv v1, v3, v2\n"
481 "vxor.vv v0, v0, v2\n"
482
483 "vsra.vi v6, v5, 7\n"
484 "vsll.vi v7, v5, 1\n"
485 "vand.vx v6, v6, %[x1d]\n"
486 "vxor.vv v7, v7, v6\n"
487 "vle8.v v6, (%[wd1])\n"
488 "vxor.vv v5, v7, v6\n"
489 "vxor.vv v4, v4, v6\n"
490
491 "vsra.vi v10, v9, 7\n"
492 "vsll.vi v11, v9, 1\n"
493 "vand.vx v10, v10, %[x1d]\n"
494 "vxor.vv v11, v11, v10\n"
495 "vle8.v v10, (%[wd2])\n"
496 "vxor.vv v9, v11, v10\n"
497 "vxor.vv v8, v8, v10\n"
498
499 "vsra.vi v14, v13, 7\n"
500 "vsll.vi v15, v13, 1\n"
501 "vand.vx v14, v14, %[x1d]\n"
502 "vxor.vv v15, v15, v14\n"
503 "vle8.v v14, (%[wd3])\n"
504 "vxor.vv v13, v15, v14\n"
505 "vxor.vv v12, v12, v14\n"
506 ".option pop\n"
507 : :
508 [wd0]"r"(&dptr[z][d + 0 * nsize]),
509 [wd1]"r"(&dptr[z][d + 1 * nsize]),
510 [wd2]"r"(&dptr[z][d + 2 * nsize]),
511 [wd3]"r"(&dptr[z][d + 3 * nsize]),
512 [x1d]"r"(0x1d)
513 );
514 }
515
516 /*
517 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
518 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
519 */
520 asm volatile (".option push\n"
521 ".option arch,+v\n"
522 "vse8.v v0, (%[wp0])\n"
523 "vse8.v v1, (%[wq0])\n"
524 "vse8.v v4, (%[wp1])\n"
525 "vse8.v v5, (%[wq1])\n"
526 "vse8.v v8, (%[wp2])\n"
527 "vse8.v v9, (%[wq2])\n"
528 "vse8.v v12, (%[wp3])\n"
529 "vse8.v v13, (%[wq3])\n"
530 ".option pop\n"
531 : :
532 [wp0]"r"(&p[d + nsize * 0]),
533 [wq0]"r"(&q[d + nsize * 0]),
534 [wp1]"r"(&p[d + nsize * 1]),
535 [wq1]"r"(&q[d + nsize * 1]),
536 [wp2]"r"(&p[d + nsize * 2]),
537 [wq2]"r"(&q[d + nsize * 2]),
538 [wp3]"r"(&p[d + nsize * 3]),
539 [wq3]"r"(&q[d + nsize * 3])
540 );
541 }
542}
543
544static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
545 unsigned long bytes, void **ptrs)
546{
547 u8 **dptr = (u8 **)ptrs;
548 u8 *p, *q;
549 unsigned long vl, d, nsize;
550 int z, z0;
551
552 z0 = stop; /* P/Q right side optimization */
553 p = dptr[disks - 2]; /* XOR parity */
554 q = dptr[disks - 1]; /* RS syndrome */
555
556 asm volatile (".option push\n"
557 ".option arch,+v\n"
558 "vsetvli %0, x0, e8, m1, ta, ma\n"
559 ".option pop\n"
560 : "=&r" (vl)
561 );
562
563 nsize = vl;
564
565 /*
566 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
567 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
568 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
569 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
570 */
571 for (d = 0; d < bytes; d += nsize * 4) {
572 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
573 asm volatile (".option push\n"
574 ".option arch,+v\n"
575 "vle8.v v0, (%[wp0])\n"
576 "vmv.v.v v1, v0\n"
577 "vle8.v v4, (%[wp1])\n"
578 "vmv.v.v v5, v4\n"
579 "vle8.v v8, (%[wp2])\n"
580 "vmv.v.v v9, v8\n"
581 "vle8.v v12, (%[wp3])\n"
582 "vmv.v.v v13, v12\n"
583 ".option pop\n"
584 : :
585 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
586 [wp1]"r"(&dptr[z0][d + 1 * nsize]),
587 [wp2]"r"(&dptr[z0][d + 2 * nsize]),
588 [wp3]"r"(&dptr[z0][d + 3 * nsize])
589 );
590
591 /* P/Q data pages */
592 for (z = z0 - 1; z >= start; z--) {
593 /*
594 * w2$$ = MASK(wq$$);
595 * w1$$ = SHLBYTE(wq$$);
596 * w2$$ &= NBYTES(0x1d);
597 * w1$$ ^= w2$$;
598 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
599 * wq$$ = w1$$ ^ wd$$;
600 * wp$$ ^= wd$$;
601 */
602 asm volatile (".option push\n"
603 ".option arch,+v\n"
604 "vsra.vi v2, v1, 7\n"
605 "vsll.vi v3, v1, 1\n"
606 "vand.vx v2, v2, %[x1d]\n"
607 "vxor.vv v3, v3, v2\n"
608 "vle8.v v2, (%[wd0])\n"
609 "vxor.vv v1, v3, v2\n"
610 "vxor.vv v0, v0, v2\n"
611
612 "vsra.vi v6, v5, 7\n"
613 "vsll.vi v7, v5, 1\n"
614 "vand.vx v6, v6, %[x1d]\n"
615 "vxor.vv v7, v7, v6\n"
616 "vle8.v v6, (%[wd1])\n"
617 "vxor.vv v5, v7, v6\n"
618 "vxor.vv v4, v4, v6\n"
619
620 "vsra.vi v10, v9, 7\n"
621 "vsll.vi v11, v9, 1\n"
622 "vand.vx v10, v10, %[x1d]\n"
623 "vxor.vv v11, v11, v10\n"
624 "vle8.v v10, (%[wd2])\n"
625 "vxor.vv v9, v11, v10\n"
626 "vxor.vv v8, v8, v10\n"
627
628 "vsra.vi v14, v13, 7\n"
629 "vsll.vi v15, v13, 1\n"
630 "vand.vx v14, v14, %[x1d]\n"
631 "vxor.vv v15, v15, v14\n"
632 "vle8.v v14, (%[wd3])\n"
633 "vxor.vv v13, v15, v14\n"
634 "vxor.vv v12, v12, v14\n"
635 ".option pop\n"
636 : :
637 [wd0]"r"(&dptr[z][d + 0 * nsize]),
638 [wd1]"r"(&dptr[z][d + 1 * nsize]),
639 [wd2]"r"(&dptr[z][d + 2 * nsize]),
640 [wd3]"r"(&dptr[z][d + 3 * nsize]),
641 [x1d]"r"(0x1d)
642 );
643 }
644
645 /* P/Q left side optimization */
646 for (z = start - 1; z >= 0; z--) {
647 /*
648 * w2$$ = MASK(wq$$);
649 * w1$$ = SHLBYTE(wq$$);
650 * w2$$ &= NBYTES(0x1d);
651 * wq$$ = w1$$ ^ w2$$;
652 */
653 asm volatile (".option push\n"
654 ".option arch,+v\n"
655 "vsra.vi v2, v1, 7\n"
656 "vsll.vi v3, v1, 1\n"
657 "vand.vx v2, v2, %[x1d]\n"
658 "vxor.vv v1, v3, v2\n"
659
660 "vsra.vi v6, v5, 7\n"
661 "vsll.vi v7, v5, 1\n"
662 "vand.vx v6, v6, %[x1d]\n"
663 "vxor.vv v5, v7, v6\n"
664
665 "vsra.vi v10, v9, 7\n"
666 "vsll.vi v11, v9, 1\n"
667 "vand.vx v10, v10, %[x1d]\n"
668 "vxor.vv v9, v11, v10\n"
669
670 "vsra.vi v14, v13, 7\n"
671 "vsll.vi v15, v13, 1\n"
672 "vand.vx v14, v14, %[x1d]\n"
673 "vxor.vv v13, v15, v14\n"
674 ".option pop\n"
675 : :
676 [x1d]"r"(0x1d)
677 );
678 }
679
680 /*
681 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
682 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
683 * v0:wp0, v1:wq0, v2:p0, v3:q0
684 * v4:wp1, v5:wq1, v6:p1, v7:q1
685 * v8:wp2, v9:wq2, v10:p2, v11:q2
686 * v12:wp3, v13:wq3, v14:p3, v15:q3
687 */
688 asm volatile (".option push\n"
689 ".option arch,+v\n"
690 "vle8.v v2, (%[wp0])\n"
691 "vle8.v v3, (%[wq0])\n"
692 "vxor.vv v2, v2, v0\n"
693 "vxor.vv v3, v3, v1\n"
694 "vse8.v v2, (%[wp0])\n"
695 "vse8.v v3, (%[wq0])\n"
696
697 "vle8.v v6, (%[wp1])\n"
698 "vle8.v v7, (%[wq1])\n"
699 "vxor.vv v6, v6, v4\n"
700 "vxor.vv v7, v7, v5\n"
701 "vse8.v v6, (%[wp1])\n"
702 "vse8.v v7, (%[wq1])\n"
703
704 "vle8.v v10, (%[wp2])\n"
705 "vle8.v v11, (%[wq2])\n"
706 "vxor.vv v10, v10, v8\n"
707 "vxor.vv v11, v11, v9\n"
708 "vse8.v v10, (%[wp2])\n"
709 "vse8.v v11, (%[wq2])\n"
710
711 "vle8.v v14, (%[wp3])\n"
712 "vle8.v v15, (%[wq3])\n"
713 "vxor.vv v14, v14, v12\n"
714 "vxor.vv v15, v15, v13\n"
715 "vse8.v v14, (%[wp3])\n"
716 "vse8.v v15, (%[wq3])\n"
717 ".option pop\n"
718 : :
719 [wp0]"r"(&p[d + nsize * 0]),
720 [wq0]"r"(&q[d + nsize * 0]),
721 [wp1]"r"(&p[d + nsize * 1]),
722 [wq1]"r"(&q[d + nsize * 1]),
723 [wp2]"r"(&p[d + nsize * 2]),
724 [wq2]"r"(&q[d + nsize * 2]),
725 [wp3]"r"(&p[d + nsize * 3]),
726 [wq3]"r"(&q[d + nsize * 3])
727 );
728 }
729}
730
731static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
732{
733 u8 **dptr = (u8 **)ptrs;
734 u8 *p, *q;
735 unsigned long vl, d, nsize;
736 int z, z0;
737
738 z0 = disks - 3; /* Highest data disk */
739 p = dptr[z0 + 1]; /* XOR parity */
740 q = dptr[z0 + 2]; /* RS syndrome */
741
742 asm volatile (".option push\n"
743 ".option arch,+v\n"
744 "vsetvli %0, x0, e8, m1, ta, ma\n"
745 ".option pop\n"
746 : "=&r" (vl)
747 );
748
749 nsize = vl;
750
751 /*
752 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
753 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
754 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
755 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
756 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
757 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
758 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
759 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
760 */
761 for (d = 0; d < bytes; d += nsize * 8) {
762 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
763 asm volatile (".option push\n"
764 ".option arch,+v\n"
765 "vle8.v v0, (%[wp0])\n"
766 "vmv.v.v v1, v0\n"
767 "vle8.v v4, (%[wp1])\n"
768 "vmv.v.v v5, v4\n"
769 "vle8.v v8, (%[wp2])\n"
770 "vmv.v.v v9, v8\n"
771 "vle8.v v12, (%[wp3])\n"
772 "vmv.v.v v13, v12\n"
773 "vle8.v v16, (%[wp4])\n"
774 "vmv.v.v v17, v16\n"
775 "vle8.v v20, (%[wp5])\n"
776 "vmv.v.v v21, v20\n"
777 "vle8.v v24, (%[wp6])\n"
778 "vmv.v.v v25, v24\n"
779 "vle8.v v28, (%[wp7])\n"
780 "vmv.v.v v29, v28\n"
781 ".option pop\n"
782 : :
783 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
784 [wp1]"r"(&dptr[z0][d + 1 * nsize]),
785 [wp2]"r"(&dptr[z0][d + 2 * nsize]),
786 [wp3]"r"(&dptr[z0][d + 3 * nsize]),
787 [wp4]"r"(&dptr[z0][d + 4 * nsize]),
788 [wp5]"r"(&dptr[z0][d + 5 * nsize]),
789 [wp6]"r"(&dptr[z0][d + 6 * nsize]),
790 [wp7]"r"(&dptr[z0][d + 7 * nsize])
791 );
792
793 for (z = z0 - 1; z >= 0; z--) {
794 /*
795 * w2$$ = MASK(wq$$);
796 * w1$$ = SHLBYTE(wq$$);
797 * w2$$ &= NBYTES(0x1d);
798 * w1$$ ^= w2$$;
799 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
800 * wq$$ = w1$$ ^ wd$$;
801 * wp$$ ^= wd$$;
802 */
803 asm volatile (".option push\n"
804 ".option arch,+v\n"
805 "vsra.vi v2, v1, 7\n"
806 "vsll.vi v3, v1, 1\n"
807 "vand.vx v2, v2, %[x1d]\n"
808 "vxor.vv v3, v3, v2\n"
809 "vle8.v v2, (%[wd0])\n"
810 "vxor.vv v1, v3, v2\n"
811 "vxor.vv v0, v0, v2\n"
812
813 "vsra.vi v6, v5, 7\n"
814 "vsll.vi v7, v5, 1\n"
815 "vand.vx v6, v6, %[x1d]\n"
816 "vxor.vv v7, v7, v6\n"
817 "vle8.v v6, (%[wd1])\n"
818 "vxor.vv v5, v7, v6\n"
819 "vxor.vv v4, v4, v6\n"
820
821 "vsra.vi v10, v9, 7\n"
822 "vsll.vi v11, v9, 1\n"
823 "vand.vx v10, v10, %[x1d]\n"
824 "vxor.vv v11, v11, v10\n"
825 "vle8.v v10, (%[wd2])\n"
826 "vxor.vv v9, v11, v10\n"
827 "vxor.vv v8, v8, v10\n"
828
829 "vsra.vi v14, v13, 7\n"
830 "vsll.vi v15, v13, 1\n"
831 "vand.vx v14, v14, %[x1d]\n"
832 "vxor.vv v15, v15, v14\n"
833 "vle8.v v14, (%[wd3])\n"
834 "vxor.vv v13, v15, v14\n"
835 "vxor.vv v12, v12, v14\n"
836
837 "vsra.vi v18, v17, 7\n"
838 "vsll.vi v19, v17, 1\n"
839 "vand.vx v18, v18, %[x1d]\n"
840 "vxor.vv v19, v19, v18\n"
841 "vle8.v v18, (%[wd4])\n"
842 "vxor.vv v17, v19, v18\n"
843 "vxor.vv v16, v16, v18\n"
844
845 "vsra.vi v22, v21, 7\n"
846 "vsll.vi v23, v21, 1\n"
847 "vand.vx v22, v22, %[x1d]\n"
848 "vxor.vv v23, v23, v22\n"
849 "vle8.v v22, (%[wd5])\n"
850 "vxor.vv v21, v23, v22\n"
851 "vxor.vv v20, v20, v22\n"
852
853 "vsra.vi v26, v25, 7\n"
854 "vsll.vi v27, v25, 1\n"
855 "vand.vx v26, v26, %[x1d]\n"
856 "vxor.vv v27, v27, v26\n"
857 "vle8.v v26, (%[wd6])\n"
858 "vxor.vv v25, v27, v26\n"
859 "vxor.vv v24, v24, v26\n"
860
861 "vsra.vi v30, v29, 7\n"
862 "vsll.vi v31, v29, 1\n"
863 "vand.vx v30, v30, %[x1d]\n"
864 "vxor.vv v31, v31, v30\n"
865 "vle8.v v30, (%[wd7])\n"
866 "vxor.vv v29, v31, v30\n"
867 "vxor.vv v28, v28, v30\n"
868 ".option pop\n"
869 : :
870 [wd0]"r"(&dptr[z][d + 0 * nsize]),
871 [wd1]"r"(&dptr[z][d + 1 * nsize]),
872 [wd2]"r"(&dptr[z][d + 2 * nsize]),
873 [wd3]"r"(&dptr[z][d + 3 * nsize]),
874 [wd4]"r"(&dptr[z][d + 4 * nsize]),
875 [wd5]"r"(&dptr[z][d + 5 * nsize]),
876 [wd6]"r"(&dptr[z][d + 6 * nsize]),
877 [wd7]"r"(&dptr[z][d + 7 * nsize]),
878 [x1d]"r"(0x1d)
879 );
880 }
881
882 /*
883 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
884 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
885 */
886 asm volatile (".option push\n"
887 ".option arch,+v\n"
888 "vse8.v v0, (%[wp0])\n"
889 "vse8.v v1, (%[wq0])\n"
890 "vse8.v v4, (%[wp1])\n"
891 "vse8.v v5, (%[wq1])\n"
892 "vse8.v v8, (%[wp2])\n"
893 "vse8.v v9, (%[wq2])\n"
894 "vse8.v v12, (%[wp3])\n"
895 "vse8.v v13, (%[wq3])\n"
896 "vse8.v v16, (%[wp4])\n"
897 "vse8.v v17, (%[wq4])\n"
898 "vse8.v v20, (%[wp5])\n"
899 "vse8.v v21, (%[wq5])\n"
900 "vse8.v v24, (%[wp6])\n"
901 "vse8.v v25, (%[wq6])\n"
902 "vse8.v v28, (%[wp7])\n"
903 "vse8.v v29, (%[wq7])\n"
904 ".option pop\n"
905 : :
906 [wp0]"r"(&p[d + nsize * 0]),
907 [wq0]"r"(&q[d + nsize * 0]),
908 [wp1]"r"(&p[d + nsize * 1]),
909 [wq1]"r"(&q[d + nsize * 1]),
910 [wp2]"r"(&p[d + nsize * 2]),
911 [wq2]"r"(&q[d + nsize * 2]),
912 [wp3]"r"(&p[d + nsize * 3]),
913 [wq3]"r"(&q[d + nsize * 3]),
914 [wp4]"r"(&p[d + nsize * 4]),
915 [wq4]"r"(&q[d + nsize * 4]),
916 [wp5]"r"(&p[d + nsize * 5]),
917 [wq5]"r"(&q[d + nsize * 5]),
918 [wp6]"r"(&p[d + nsize * 6]),
919 [wq6]"r"(&q[d + nsize * 6]),
920 [wp7]"r"(&p[d + nsize * 7]),
921 [wq7]"r"(&q[d + nsize * 7])
922 );
923 }
924}
925
926static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
927 unsigned long bytes, void **ptrs)
928{
929 u8 **dptr = (u8 **)ptrs;
930 u8 *p, *q;
931 unsigned long vl, d, nsize;
932 int z, z0;
933
934 z0 = stop; /* P/Q right side optimization */
935 p = dptr[disks - 2]; /* XOR parity */
936 q = dptr[disks - 1]; /* RS syndrome */
937
938 asm volatile (".option push\n"
939 ".option arch,+v\n"
940 "vsetvli %0, x0, e8, m1, ta, ma\n"
941 ".option pop\n"
942 : "=&r" (vl)
943 );
944
945 nsize = vl;
946
947 /*
948 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
949 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
950 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
951 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
952 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
953 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
954 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
955 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
956 */
957 for (d = 0; d < bytes; d += nsize * 8) {
958 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
959 asm volatile (".option push\n"
960 ".option arch,+v\n"
961 "vle8.v v0, (%[wp0])\n"
962 "vmv.v.v v1, v0\n"
963 "vle8.v v4, (%[wp1])\n"
964 "vmv.v.v v5, v4\n"
965 "vle8.v v8, (%[wp2])\n"
966 "vmv.v.v v9, v8\n"
967 "vle8.v v12, (%[wp3])\n"
968 "vmv.v.v v13, v12\n"
969 "vle8.v v16, (%[wp4])\n"
970 "vmv.v.v v17, v16\n"
971 "vle8.v v20, (%[wp5])\n"
972 "vmv.v.v v21, v20\n"
973 "vle8.v v24, (%[wp6])\n"
974 "vmv.v.v v25, v24\n"
975 "vle8.v v28, (%[wp7])\n"
976 "vmv.v.v v29, v28\n"
977 ".option pop\n"
978 : :
979 [wp0]"r"(&dptr[z0][d + 0 * nsize]),
980 [wp1]"r"(&dptr[z0][d + 1 * nsize]),
981 [wp2]"r"(&dptr[z0][d + 2 * nsize]),
982 [wp3]"r"(&dptr[z0][d + 3 * nsize]),
983 [wp4]"r"(&dptr[z0][d + 4 * nsize]),
984 [wp5]"r"(&dptr[z0][d + 5 * nsize]),
985 [wp6]"r"(&dptr[z0][d + 6 * nsize]),
986 [wp7]"r"(&dptr[z0][d + 7 * nsize])
987 );
988
989 /* P/Q data pages */
990 for (z = z0 - 1; z >= start; z--) {
991 /*
992 * w2$$ = MASK(wq$$);
993 * w1$$ = SHLBYTE(wq$$);
994 * w2$$ &= NBYTES(0x1d);
995 * w1$$ ^= w2$$;
996 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
997 * wq$$ = w1$$ ^ wd$$;
998 * wp$$ ^= wd$$;
999 */
1000 asm volatile (".option push\n"
1001 ".option arch,+v\n"
1002 "vsra.vi v2, v1, 7\n"
1003 "vsll.vi v3, v1, 1\n"
1004 "vand.vx v2, v2, %[x1d]\n"
1005 "vxor.vv v3, v3, v2\n"
1006 "vle8.v v2, (%[wd0])\n"
1007 "vxor.vv v1, v3, v2\n"
1008 "vxor.vv v0, v0, v2\n"
1009
1010 "vsra.vi v6, v5, 7\n"
1011 "vsll.vi v7, v5, 1\n"
1012 "vand.vx v6, v6, %[x1d]\n"
1013 "vxor.vv v7, v7, v6\n"
1014 "vle8.v v6, (%[wd1])\n"
1015 "vxor.vv v5, v7, v6\n"
1016 "vxor.vv v4, v4, v6\n"
1017
1018 "vsra.vi v10, v9, 7\n"
1019 "vsll.vi v11, v9, 1\n"
1020 "vand.vx v10, v10, %[x1d]\n"
1021 "vxor.vv v11, v11, v10\n"
1022 "vle8.v v10, (%[wd2])\n"
1023 "vxor.vv v9, v11, v10\n"
1024 "vxor.vv v8, v8, v10\n"
1025
1026 "vsra.vi v14, v13, 7\n"
1027 "vsll.vi v15, v13, 1\n"
1028 "vand.vx v14, v14, %[x1d]\n"
1029 "vxor.vv v15, v15, v14\n"
1030 "vle8.v v14, (%[wd3])\n"
1031 "vxor.vv v13, v15, v14\n"
1032 "vxor.vv v12, v12, v14\n"
1033
1034 "vsra.vi v18, v17, 7\n"
1035 "vsll.vi v19, v17, 1\n"
1036 "vand.vx v18, v18, %[x1d]\n"
1037 "vxor.vv v19, v19, v18\n"
1038 "vle8.v v18, (%[wd4])\n"
1039 "vxor.vv v17, v19, v18\n"
1040 "vxor.vv v16, v16, v18\n"
1041
1042 "vsra.vi v22, v21, 7\n"
1043 "vsll.vi v23, v21, 1\n"
1044 "vand.vx v22, v22, %[x1d]\n"
1045 "vxor.vv v23, v23, v22\n"
1046 "vle8.v v22, (%[wd5])\n"
1047 "vxor.vv v21, v23, v22\n"
1048 "vxor.vv v20, v20, v22\n"
1049
1050 "vsra.vi v26, v25, 7\n"
1051 "vsll.vi v27, v25, 1\n"
1052 "vand.vx v26, v26, %[x1d]\n"
1053 "vxor.vv v27, v27, v26\n"
1054 "vle8.v v26, (%[wd6])\n"
1055 "vxor.vv v25, v27, v26\n"
1056 "vxor.vv v24, v24, v26\n"
1057
1058 "vsra.vi v30, v29, 7\n"
1059 "vsll.vi v31, v29, 1\n"
1060 "vand.vx v30, v30, %[x1d]\n"
1061 "vxor.vv v31, v31, v30\n"
1062 "vle8.v v30, (%[wd7])\n"
1063 "vxor.vv v29, v31, v30\n"
1064 "vxor.vv v28, v28, v30\n"
1065 ".option pop\n"
1066 : :
1067 [wd0]"r"(&dptr[z][d + 0 * nsize]),
1068 [wd1]"r"(&dptr[z][d + 1 * nsize]),
1069 [wd2]"r"(&dptr[z][d + 2 * nsize]),
1070 [wd3]"r"(&dptr[z][d + 3 * nsize]),
1071 [wd4]"r"(&dptr[z][d + 4 * nsize]),
1072 [wd5]"r"(&dptr[z][d + 5 * nsize]),
1073 [wd6]"r"(&dptr[z][d + 6 * nsize]),
1074 [wd7]"r"(&dptr[z][d + 7 * nsize]),
1075 [x1d]"r"(0x1d)
1076 );
1077 }
1078
1079 /* P/Q left side optimization */
1080 for (z = start - 1; z >= 0; z--) {
1081 /*
1082 * w2$$ = MASK(wq$$);
1083 * w1$$ = SHLBYTE(wq$$);
1084 * w2$$ &= NBYTES(0x1d);
1085 * wq$$ = w1$$ ^ w2$$;
1086 */
1087 asm volatile (".option push\n"
1088 ".option arch,+v\n"
1089 "vsra.vi v2, v1, 7\n"
1090 "vsll.vi v3, v1, 1\n"
1091 "vand.vx v2, v2, %[x1d]\n"
1092 "vxor.vv v1, v3, v2\n"
1093
1094 "vsra.vi v6, v5, 7\n"
1095 "vsll.vi v7, v5, 1\n"
1096 "vand.vx v6, v6, %[x1d]\n"
1097 "vxor.vv v5, v7, v6\n"
1098
1099 "vsra.vi v10, v9, 7\n"
1100 "vsll.vi v11, v9, 1\n"
1101 "vand.vx v10, v10, %[x1d]\n"
1102 "vxor.vv v9, v11, v10\n"
1103
1104 "vsra.vi v14, v13, 7\n"
1105 "vsll.vi v15, v13, 1\n"
1106 "vand.vx v14, v14, %[x1d]\n"
1107 "vxor.vv v13, v15, v14\n"
1108
1109 "vsra.vi v18, v17, 7\n"
1110 "vsll.vi v19, v17, 1\n"
1111 "vand.vx v18, v18, %[x1d]\n"
1112 "vxor.vv v17, v19, v18\n"
1113
1114 "vsra.vi v22, v21, 7\n"
1115 "vsll.vi v23, v21, 1\n"
1116 "vand.vx v22, v22, %[x1d]\n"
1117 "vxor.vv v21, v23, v22\n"
1118
1119 "vsra.vi v26, v25, 7\n"
1120 "vsll.vi v27, v25, 1\n"
1121 "vand.vx v26, v26, %[x1d]\n"
1122 "vxor.vv v25, v27, v26\n"
1123
1124 "vsra.vi v30, v29, 7\n"
1125 "vsll.vi v31, v29, 1\n"
1126 "vand.vx v30, v30, %[x1d]\n"
1127 "vxor.vv v29, v31, v30\n"
1128 ".option pop\n"
1129 : :
1130 [x1d]"r"(0x1d)
1131 );
1132 }
1133
1134 /*
1135 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
1136 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
1137 * v0:wp0, v1:wq0, v2:p0, v3:q0
1138 * v4:wp1, v5:wq1, v6:p1, v7:q1
1139 * v8:wp2, v9:wq2, v10:p2, v11:q2
1140 * v12:wp3, v13:wq3, v14:p3, v15:q3
1141 * v16:wp4, v17:wq4, v18:p4, v19:q4
1142 * v20:wp5, v21:wq5, v22:p5, v23:q5
1143 * v24:wp6, v25:wq6, v26:p6, v27:q6
1144 * v28:wp7, v29:wq7, v30:p7, v31:q7
1145 */
1146 asm volatile (".option push\n"
1147 ".option arch,+v\n"
1148 "vle8.v v2, (%[wp0])\n"
1149 "vle8.v v3, (%[wq0])\n"
1150 "vxor.vv v2, v2, v0\n"
1151 "vxor.vv v3, v3, v1\n"
1152 "vse8.v v2, (%[wp0])\n"
1153 "vse8.v v3, (%[wq0])\n"
1154
1155 "vle8.v v6, (%[wp1])\n"
1156 "vle8.v v7, (%[wq1])\n"
1157 "vxor.vv v6, v6, v4\n"
1158 "vxor.vv v7, v7, v5\n"
1159 "vse8.v v6, (%[wp1])\n"
1160 "vse8.v v7, (%[wq1])\n"
1161
1162 "vle8.v v10, (%[wp2])\n"
1163 "vle8.v v11, (%[wq2])\n"
1164 "vxor.vv v10, v10, v8\n"
1165 "vxor.vv v11, v11, v9\n"
1166 "vse8.v v10, (%[wp2])\n"
1167 "vse8.v v11, (%[wq2])\n"
1168
1169 "vle8.v v14, (%[wp3])\n"
1170 "vle8.v v15, (%[wq3])\n"
1171 "vxor.vv v14, v14, v12\n"
1172 "vxor.vv v15, v15, v13\n"
1173 "vse8.v v14, (%[wp3])\n"
1174 "vse8.v v15, (%[wq3])\n"
1175
1176 "vle8.v v18, (%[wp4])\n"
1177 "vle8.v v19, (%[wq4])\n"
1178 "vxor.vv v18, v18, v16\n"
1179 "vxor.vv v19, v19, v17\n"
1180 "vse8.v v18, (%[wp4])\n"
1181 "vse8.v v19, (%[wq4])\n"
1182
1183 "vle8.v v22, (%[wp5])\n"
1184 "vle8.v v23, (%[wq5])\n"
1185 "vxor.vv v22, v22, v20\n"
1186 "vxor.vv v23, v23, v21\n"
1187 "vse8.v v22, (%[wp5])\n"
1188 "vse8.v v23, (%[wq5])\n"
1189
1190 "vle8.v v26, (%[wp6])\n"
1191 "vle8.v v27, (%[wq6])\n"
1192 "vxor.vv v26, v26, v24\n"
1193 "vxor.vv v27, v27, v25\n"
1194 "vse8.v v26, (%[wp6])\n"
1195 "vse8.v v27, (%[wq6])\n"
1196
1197 "vle8.v v30, (%[wp7])\n"
1198 "vle8.v v31, (%[wq7])\n"
1199 "vxor.vv v30, v30, v28\n"
1200 "vxor.vv v31, v31, v29\n"
1201 "vse8.v v30, (%[wp7])\n"
1202 "vse8.v v31, (%[wq7])\n"
1203 ".option pop\n"
1204 : :
1205 [wp0]"r"(&p[d + nsize * 0]),
1206 [wq0]"r"(&q[d + nsize * 0]),
1207 [wp1]"r"(&p[d + nsize * 1]),
1208 [wq1]"r"(&q[d + nsize * 1]),
1209 [wp2]"r"(&p[d + nsize * 2]),
1210 [wq2]"r"(&q[d + nsize * 2]),
1211 [wp3]"r"(&p[d + nsize * 3]),
1212 [wq3]"r"(&q[d + nsize * 3]),
1213 [wp4]"r"(&p[d + nsize * 4]),
1214 [wq4]"r"(&q[d + nsize * 4]),
1215 [wp5]"r"(&p[d + nsize * 5]),
1216 [wq5]"r"(&q[d + nsize * 5]),
1217 [wp6]"r"(&p[d + nsize * 6]),
1218 [wq6]"r"(&q[d + nsize * 6]),
1219 [wp7]"r"(&p[d + nsize * 7]),
1220 [wq7]"r"(&q[d + nsize * 7])
1221 );
1222 }
1223}
1224
1225RAID6_RVV_WRAPPER(1);
1226RAID6_RVV_WRAPPER(2);
1227RAID6_RVV_WRAPPER(4);
1228RAID6_RVV_WRAPPER(8);