tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Calculate a CRC T10-DIF with vpmsum acceleration
4 *
5 * Copyright 2017, Daniel Axtens, IBM Corporation.
6 * [based on crc32c-vpmsum_glue.c]
7 */
8
9#include <asm/switch_to.h>
10#include <crypto/internal/simd.h>
11#include <linux/cpufeature.h>
12#include <linux/crc-t10dif.h>
13#include <linux/jump_label.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/preempt.h>
17#include <linux/uaccess.h>
18
19#define VMX_ALIGN 16
20#define VMX_ALIGN_MASK (VMX_ALIGN-1)
21
22#define VECTOR_BREAKPOINT 64
23
24static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
25
26u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
27
28u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
29{
30 unsigned int prealign;
31 unsigned int tail;
32 u32 crc = crci;
33
34 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
35 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
36 return crc_t10dif_generic(crc, p, len);
37
38 if ((unsigned long)p & VMX_ALIGN_MASK) {
39 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
40 crc = crc_t10dif_generic(crc, p, prealign);
41 len -= prealign;
42 p += prealign;
43 }
44
45 if (len & ~VMX_ALIGN_MASK) {
46 crc <<= 16;
47 preempt_disable();
48 pagefault_disable();
49 enable_kernel_altivec();
50 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
51 disable_kernel_altivec();
52 pagefault_enable();
53 preempt_enable();
54 crc >>= 16;
55 }
56
57 tail = len & VMX_ALIGN_MASK;
58 if (tail) {
59 p += len & ~VMX_ALIGN_MASK;
60 crc = crc_t10dif_generic(crc, p, tail);
61 }
62
63 return crc & 0xffff;
64}
65EXPORT_SYMBOL(crc_t10dif_arch);
66
67static int __init crc_t10dif_powerpc_init(void)
68{
69 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
70 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
71 static_branch_enable(&have_vec_crypto);
72 return 0;
73}
74subsys_initcall(crc_t10dif_powerpc_init);
75
76static void __exit crc_t10dif_powerpc_exit(void)
77{
78}
79module_exit(crc_t10dif_powerpc_exit);
80
81MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
82MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
83MODULE_LICENSE("GPL");