tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/crc32.h>
3#include <crypto/internal/simd.h>
4#include <linux/init.h>
5#include <linux/module.h>
6#include <linux/kernel.h>
7#include <linux/cpufeature.h>
8#include <asm/simd.h>
9#include <asm/switch_to.h>
10
11#define VMX_ALIGN 16
12#define VMX_ALIGN_MASK (VMX_ALIGN-1)
13
14#define VECTOR_BREAKPOINT 512
15
16static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
17
18u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
19
20u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21{
22 return crc32_le_base(crc, p, len);
23}
24EXPORT_SYMBOL(crc32_le_arch);
25
26u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
27{
28 unsigned int prealign;
29 unsigned int tail;
30
31 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33 return crc32c_base(crc, p, len);
34
35 if ((unsigned long)p & VMX_ALIGN_MASK) {
36 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37 crc = crc32c_base(crc, p, prealign);
38 len -= prealign;
39 p += prealign;
40 }
41
42 if (len & ~VMX_ALIGN_MASK) {
43 preempt_disable();
44 pagefault_disable();
45 enable_kernel_altivec();
46 crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
47 disable_kernel_altivec();
48 pagefault_enable();
49 preempt_enable();
50 }
51
52 tail = len & VMX_ALIGN_MASK;
53 if (tail) {
54 p += len & ~VMX_ALIGN_MASK;
55 crc = crc32c_base(crc, p, tail);
56 }
57
58 return crc;
59}
60EXPORT_SYMBOL(crc32c_arch);
61
62u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
63{
64 return crc32_be_base(crc, p, len);
65}
66EXPORT_SYMBOL(crc32_be_arch);
67
68static int __init crc32_powerpc_init(void)
69{
70 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
71 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
72 static_branch_enable(&have_vec_crypto);
73 return 0;
74}
75arch_initcall(crc32_powerpc_init);
76
77static void __exit crc32_powerpc_exit(void)
78{
79}
80module_exit(crc32_powerpc_exit);
81
82u32 crc32_optimizations(void)
83{
84 if (static_key_enabled(&have_vec_crypto))
85 return CRC32C_OPTIMIZATION;
86 return 0;
87}
88EXPORT_SYMBOL(crc32_optimizations);
89
90MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
91MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
92MODULE_LICENSE("GPL");