tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8#include <crypto/internal/simd.h>
9
10#include <asm/neon.h>
11#include <asm/simd.h>
12
13static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
14static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
15
16#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
17
18asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
19asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
20 u8 out[16]);
21
22static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
23{
24 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
25 if (static_branch_likely(&have_pmull)) {
26 if (crypto_simd_usable()) {
27 kernel_neon_begin();
28 crc = crc_t10dif_pmull64(crc, data, length);
29 kernel_neon_end();
30 return crc;
31 }
32 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
33 static_branch_likely(&have_neon) &&
34 crypto_simd_usable()) {
35 u8 buf[16] __aligned(16);
36
37 kernel_neon_begin();
38 crc_t10dif_pmull8(crc, data, length, buf);
39 kernel_neon_end();
40
41 return crc_t10dif_generic(0, buf, sizeof(buf));
42 }
43 }
44 return crc_t10dif_generic(crc, data, length);
45}
46
47#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
48static inline void crc_t10dif_mod_init_arch(void)
49{
50 if (elf_hwcap & HWCAP_NEON) {
51 static_branch_enable(&have_neon);
52 if (elf_hwcap2 & HWCAP2_PMULL)
53 static_branch_enable(&have_pmull);
54 }
55}