tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

lib/crypto: chacha: Consolidate into single module

Consolidate the ChaCha code into a single module (excluding
chacha-block-generic.c which remains always built-in for random.c),
similar to various other algorithms:

- Each arch now provides a header file lib/crypto/$(SRCARCH)/chacha.h,
replacing lib/crypto/$(SRCARCH)/chacha*.c. The header defines
chacha_crypt_arch() and hchacha_block_arch(). It is included by
lib/crypto/chacha.c, and thus the code gets built into the single
libchacha module, with improved inlining in some cases.

- Whether arch-optimized ChaCha is buildable is now controlled centrally
by lib/crypto/Kconfig instead of by lib/crypto/$(SRCARCH)/Kconfig.
The conditions for enabling it remain the same as before, and it
remains enabled by default.

- Any additional arch-specific translation units for the optimized
ChaCha code, such as assembly files, are now compiled by
lib/crypto/Makefile instead of lib/crypto/$(SRCARCH)/Makefile.

This removes the last use for the Makefile and Kconfig files in the
arm64, mips, powerpc, riscv, and s390 subdirectories of lib/crypto/. So
also remove those files and the references to them.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250827151131.27733-7-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

Eric Biggers 13cecc52 1ae46b6e

+127 -290
+4 -24
include/crypto/chacha.h
··· 45 45 chacha_block_generic(state, out, 20); 46 46 } 47 47 48 - void hchacha_block_arch(const struct chacha_state *state, 49 - u32 out[HCHACHA_OUT_WORDS], int nrounds); 50 48 void hchacha_block_generic(const struct chacha_state *state, 51 49 u32 out[HCHACHA_OUT_WORDS], int nrounds); 52 50 53 - static inline void hchacha_block(const struct chacha_state *state, 54 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 55 - { 56 - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) 57 - hchacha_block_arch(state, out, nrounds); 58 - else 59 - hchacha_block_generic(state, out, nrounds); 60 - } 51 + void hchacha_block(const struct chacha_state *state, 52 + u32 out[HCHACHA_OUT_WORDS], int nrounds); 61 53 62 54 enum chacha_constants { /* expand 32-byte k */ 63 55 CHACHA_CONSTANT_EXPA = 0x61707865U, ··· 85 93 state->x[15] = get_unaligned_le32(iv + 12); 86 94 } 87 95 88 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 89 - unsigned int bytes, int nrounds); 90 - void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, 91 - unsigned int bytes, int nrounds); 92 - 93 - static inline void chacha_crypt(struct chacha_state *state, 94 - u8 *dst, const u8 *src, 95 - unsigned int bytes, int nrounds) 96 - { 97 - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) 98 - chacha_crypt_arch(state, dst, src, bytes, nrounds); 99 - else 100 - chacha_crypt_generic(state, dst, src, bytes, nrounds); 101 - } 96 + void chacha_crypt(struct chacha_state *state, u8 *dst, const u8 *src, 97 + unsigned int bytes, int nrounds); 102 98 103 99 static inline void chacha20_crypt(struct chacha_state *state, 104 100 u8 *dst, const u8 *src, unsigned int bytes)
+14 -35
lib/crypto/Kconfig
··· 44 44 implementation is enabled, this implementation serves the users 45 45 of CRYPTO_LIB_BLAKE2S. 46 46 47 - config CRYPTO_ARCH_HAVE_LIB_CHACHA 48 - bool 49 - help 50 - Declares whether the architecture provides an arch-specific 51 - accelerated implementation of the ChaCha library interface, 52 - either builtin or as a module. 53 - 54 - config CRYPTO_LIB_CHACHA_GENERIC 55 - tristate 56 - default CRYPTO_LIB_CHACHA if !CRYPTO_ARCH_HAVE_LIB_CHACHA 57 - select CRYPTO_LIB_UTILS 58 - help 59 - This symbol can be selected by arch implementations of the ChaCha 60 - library interface that require the generic code as a fallback, e.g., 61 - for SIMD implementations. If no arch specific implementation is 62 - enabled, this implementation serves the users of CRYPTO_LIB_CHACHA. 63 - 64 47 config CRYPTO_LIB_CHACHA 65 48 tristate 49 + select CRYPTO_LIB_UTILS 66 50 help 67 - Enable the ChaCha library interface. This interface may be fulfilled 68 - by either the generic implementation or an arch-specific one, if one 69 - is available and enabled. 51 + Enable the ChaCha library interface. Select this if your module uses 52 + chacha_crypt() or hchacha_block(). 53 + 54 + config CRYPTO_LIB_CHACHA_ARCH 55 + bool 56 + depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN 57 + default y if ARM 58 + default y if ARM64 && KERNEL_MODE_NEON 59 + default y if MIPS && CPU_MIPS32_R2 60 + default y if PPC64 && CPU_LITTLE_ENDIAN && VSX 61 + default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO 62 + default y if S390 63 + default y if X86_64 70 64 71 65 config CRYPTO_ARCH_HAVE_LIB_CURVE25519 72 66 bool ··· 211 217 if !KMSAN # avoid false positives from assembly 212 218 if ARM 213 219 source "lib/crypto/arm/Kconfig" 214 - endif 215 - if ARM64 216 - source "lib/crypto/arm64/Kconfig" 217 - endif 218 - if MIPS 219 - source "lib/crypto/mips/Kconfig" 220 - endif 221 - if PPC 222 - source "lib/crypto/powerpc/Kconfig" 223 - endif 224 - if RISCV 225 - source "lib/crypto/riscv/Kconfig" 226 - endif 227 - if S390 228 - source "lib/crypto/s390/Kconfig" 229 220 endif 230 221 if X86 231 222 source "lib/crypto/x86/Kconfig"
+33 -10
lib/crypto/Makefile
··· 15 15 obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o 16 16 libcryptoutils-y := memneq.o utils.o 17 17 18 - # chacha20_block() is used by the /dev/random driver which is always builtin 19 - obj-y += chacha-block-generic.o 20 - obj-$(CONFIG_CRYPTO_LIB_CHACHA_GENERIC) += libchacha.o 21 - libchacha-y := chacha.o 22 - 23 18 obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o 24 19 libaes-y := aes.o 25 20 ··· 34 39 libblake2s-y := blake2s.o 35 40 libblake2s-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += blake2s-generic.o 36 41 libblake2s-$(CONFIG_CRYPTO_SELFTESTS) += blake2s-selftest.o 42 + 43 + ################################################################################ 44 + 45 + # chacha20_block() is used by the /dev/random driver which is always builtin 46 + obj-y += chacha-block-generic.o 47 + 48 + obj-$(CONFIG_CRYPTO_LIB_CHACHA) += libchacha.o 49 + libchacha-y := chacha.o 50 + 51 + ifeq ($(CONFIG_CRYPTO_LIB_CHACHA_ARCH),y) 52 + CFLAGS_chacha.o += -I$(src)/$(SRCARCH) 53 + 54 + ifeq ($(CONFIG_ARM),y) 55 + libchacha-y += arm/chacha-scalar-core.o 56 + libchacha-$(CONFIG_KERNEL_MODE_NEON) += arm/chacha-neon-core.o 57 + endif 58 + 59 + libchacha-$(CONFIG_ARM64) += arm64/chacha-neon-core.o 60 + 61 + ifeq ($(CONFIG_MIPS),y) 62 + libchacha-y += mips/chacha-core.o 63 + AFLAGS_mips/chacha-core.o += -O2 # needed to fill branch delay slots 64 + endif 65 + 66 + libchacha-$(CONFIG_PPC) += powerpc/chacha-p10le-8x.o 67 + libchacha-$(CONFIG_RISCV) += riscv/chacha-riscv64-zvkb.o 68 + libchacha-$(CONFIG_S390) += s390/chacha-s390.o 69 + libchacha-$(CONFIG_X86) += x86/chacha-ssse3-x86_64.o \ 70 + x86/chacha-avx2-x86_64.o \ 71 + x86/chacha-avx512vl-x86_64.o 72 + endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH 73 + 74 + ################################################################################ 37 75 38 76 obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o 39 77 libchacha20poly1305-y += chacha20poly1305.o ··· 259 231 libsm3-y := sm3.o 260 232 261 233 obj-$(CONFIG_ARM) += arm/ 262 - obj-$(CONFIG_ARM64) += arm64/ 263 - obj-$(CONFIG_MIPS) += mips/ 264 - obj-$(CONFIG_PPC) += powerpc/ 265 - obj-$(CONFIG_RISCV) += riscv/ 266 - obj-$(CONFIG_S390) += s390/ 267 234 obj-$(CONFIG_X86) += x86/ 268 235 269 236 # clean-files must be defined unconditionally
-5
lib/crypto/arm/Kconfig
··· 12 12 BLAKE2b, but slower than the NEON implementation of BLAKE2b. 13 13 There is no NEON implementation of BLAKE2s, since NEON doesn't 14 14 really help with it. 15 - 16 - config CRYPTO_CHACHA20_NEON 17 - tristate 18 - default CRYPTO_LIB_CHACHA 19 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-4
lib/crypto/arm/Makefile
··· 2 2 3 3 obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += libblake2s-arm.o 4 4 libblake2s-arm-y := blake2s-core.o blake2s-glue.o 5 - 6 - obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o 7 - chacha-neon-y := chacha-scalar-core.o chacha-glue.o 8 - chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
+7 -21
lib/crypto/arm/chacha-glue.c lib/crypto/arm/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 /* 3 3 * ChaCha and HChaCha functions (ARM optimized) 4 4 * ··· 6 6 * Copyright (C) 2015 Martin Willi 7 7 */ 8 8 9 - #include <crypto/chacha.h> 10 9 #include <crypto/internal/simd.h> 11 10 #include <linux/jump_label.h> 12 11 #include <linux/kernel.h> 13 - #include <linux/module.h> 14 12 15 13 #include <asm/cputype.h> 16 14 #include <asm/hwcap.h> ··· 62 64 } 63 65 } 64 66 65 - void hchacha_block_arch(const struct chacha_state *state, 66 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 67 + static void hchacha_block_arch(const struct chacha_state *state, 68 + u32 out[HCHACHA_OUT_WORDS], int nrounds) 67 69 { 68 70 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { 69 71 hchacha_block_arm(state, out, nrounds); ··· 73 75 kernel_neon_end(); 74 76 } 75 77 } 76 - EXPORT_SYMBOL(hchacha_block_arch); 77 78 78 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 79 - unsigned int bytes, int nrounds) 79 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 80 + const u8 *src, unsigned int bytes, int nrounds) 80 81 { 81 82 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() || 82 83 bytes <= CHACHA_BLOCK_SIZE) { ··· 96 99 dst += todo; 97 100 } while (bytes); 98 101 } 99 - EXPORT_SYMBOL(chacha_crypt_arch); 100 102 101 - static int __init chacha_arm_mod_init(void) 103 + #define chacha_mod_init_arch chacha_mod_init_arch 104 + static void chacha_mod_init_arch(void) 102 105 { 103 106 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { 104 107 switch (read_cpuid_part()) { ··· 114 117 static_branch_enable(&use_neon); 115 118 } 116 119 } 117 - return 0; 118 120 } 119 - subsys_initcall(chacha_arm_mod_init); 120 - 121 - static void __exit chacha_arm_mod_exit(void) 122 - { 123 - } 124 - module_exit(chacha_arm_mod_exit); 125 - 126 - MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM optimized)"); 127 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 128 - MODULE_LICENSE("GPL v2");
-8
lib/crypto/arm64/Kconfig
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - config CRYPTO_CHACHA20_NEON 4 - tristate 5 - depends on KERNEL_MODE_NEON 6 - default CRYPTO_LIB_CHACHA 7 - select CRYPTO_LIB_CHACHA_GENERIC 8 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-4
lib/crypto/arm64/Makefile
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o 4 - chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
+6 -20
lib/crypto/arm64/chacha-neon-glue.c lib/crypto/arm64/chacha.h
··· 18 18 * (at your option) any later version. 19 19 */ 20 20 21 - #include <crypto/chacha.h> 22 21 #include <crypto/internal/simd.h> 23 22 #include <linux/jump_label.h> 24 23 #include <linux/kernel.h> 25 - #include <linux/module.h> 26 24 27 25 #include <asm/hwcap.h> 28 26 #include <asm/neon.h> ··· 59 61 } 60 62 } 61 63 62 - void hchacha_block_arch(const struct chacha_state *state, 63 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 64 + static void hchacha_block_arch(const struct chacha_state *state, 65 + u32 out[HCHACHA_OUT_WORDS], int nrounds) 64 66 { 65 67 if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { 66 68 hchacha_block_generic(state, out, nrounds); ··· 70 72 kernel_neon_end(); 71 73 } 72 74 } 73 - EXPORT_SYMBOL(hchacha_block_arch); 74 75 75 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 76 - unsigned int bytes, int nrounds) 76 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 77 + const u8 *src, unsigned int bytes, int nrounds) 77 78 { 78 79 if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE || 79 80 !crypto_simd_usable()) ··· 90 93 dst += todo; 91 94 } while (bytes); 92 95 } 93 - EXPORT_SYMBOL(chacha_crypt_arch); 94 96 95 - static int __init chacha_simd_mod_init(void) 97 + #define chacha_mod_init_arch chacha_mod_init_arch 98 + static void chacha_mod_init_arch(void) 96 99 { 97 100 if (cpu_have_named_feature(ASIMD)) 98 101 static_branch_enable(&have_neon); 99 - return 0; 100 102 } 101 - subsys_initcall(chacha_simd_mod_init); 102 - 103 - static void __exit chacha_simd_mod_exit(void) 104 - { 105 - } 106 - module_exit(chacha_simd_mod_exit); 107 - 108 - MODULE_DESCRIPTION("ChaCha and HChaCha functions (ARM64 optimized)"); 109 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 110 - MODULE_LICENSE("GPL v2");
+38 -3
lib/crypto/chacha.c
··· 11 11 #include <linux/kernel.h> 12 12 #include <linux/module.h> 13 13 14 - void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, 15 - unsigned int bytes, int nrounds) 14 + static void __maybe_unused 15 + chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, 16 + unsigned int bytes, int nrounds) 16 17 { 17 18 /* aligned to potentially speed up crypto_xor() */ 18 19 u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long)); ··· 30 29 crypto_xor_cpy(dst, src, stream, bytes); 31 30 } 32 31 } 33 - EXPORT_SYMBOL(chacha_crypt_generic); 32 + 33 + #ifdef CONFIG_CRYPTO_LIB_CHACHA_ARCH 34 + #include "chacha.h" /* $(SRCARCH)/chacha.h */ 35 + #else 36 + #define chacha_crypt_arch chacha_crypt_generic 37 + #define hchacha_block_arch hchacha_block_generic 38 + #endif 39 + 40 + void chacha_crypt(struct chacha_state *state, u8 *dst, const u8 *src, 41 + unsigned int bytes, int nrounds) 42 + { 43 + chacha_crypt_arch(state, dst, src, bytes, nrounds); 44 + } 45 + EXPORT_SYMBOL_GPL(chacha_crypt); 46 + 47 + void hchacha_block(const struct chacha_state *state, 48 + u32 out[HCHACHA_OUT_WORDS], int nrounds) 49 + { 50 + hchacha_block_arch(state, out, nrounds); 51 + } 52 + EXPORT_SYMBOL_GPL(hchacha_block); 53 + 54 + #ifdef chacha_mod_init_arch 55 + static int __init chacha_mod_init(void) 56 + { 57 + chacha_mod_init_arch(); 58 + return 0; 59 + } 60 + subsys_initcall(chacha_mod_init); 61 + 62 + static void __exit chacha_mod_exit(void) 63 + { 64 + } 65 + module_exit(chacha_mod_exit); 66 + #endif 34 67 35 68 MODULE_DESCRIPTION("ChaCha stream cipher (RFC7539)"); 36 69 MODULE_LICENSE("GPL");
-7
lib/crypto/mips/Kconfig
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - config CRYPTO_CHACHA_MIPS 4 - tristate 5 - depends on CPU_MIPS32_R2 6 - default CRYPTO_LIB_CHACHA 7 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-5
lib/crypto/mips/Makefile
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - obj-$(CONFIG_CRYPTO_CHACHA_MIPS) += chacha-mips.o 4 - chacha-mips-y := chacha-core.o chacha-glue.o 5 - AFLAGS_chacha-core.o += -O2 # needed to fill branch delay slots
+1 -10
lib/crypto/mips/chacha-glue.c lib/crypto/mips/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 /* 3 3 * ChaCha and HChaCha functions (MIPS optimized) 4 4 * 5 5 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> 6 6 */ 7 7 8 - #include <crypto/chacha.h> 9 8 #include <linux/kernel.h> 10 - #include <linux/module.h> 11 9 12 10 asmlinkage void chacha_crypt_arch(struct chacha_state *state, 13 11 u8 *dst, const u8 *src, 14 12 unsigned int bytes, int nrounds); 15 - EXPORT_SYMBOL(chacha_crypt_arch); 16 - 17 13 asmlinkage void hchacha_block_arch(const struct chacha_state *state, 18 14 u32 out[HCHACHA_OUT_WORDS], int nrounds); 19 - EXPORT_SYMBOL(hchacha_block_arch); 20 - 21 - MODULE_DESCRIPTION("ChaCha and HChaCha functions (MIPS optimized)"); 22 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 23 - MODULE_LICENSE("GPL v2");
-8
lib/crypto/powerpc/Kconfig
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - config CRYPTO_CHACHA20_P10 4 - tristate 5 - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX 6 - default CRYPTO_LIB_CHACHA 7 - select CRYPTO_LIB_CHACHA_GENERIC 8 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-4
lib/crypto/powerpc/Makefile
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - obj-$(CONFIG_CRYPTO_CHACHA20_P10) += chacha-p10-crypto.o 4 - chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o
+6 -24
lib/crypto/powerpc/chacha-p10-glue.c lib/crypto/powerpc/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 /* 3 3 * ChaCha stream cipher (P10 accelerated) 4 4 * 5 5 * Copyright 2023- IBM Corp. All rights reserved. 6 6 */ 7 7 8 - #include <crypto/chacha.h> 9 8 #include <crypto/internal/simd.h> 10 9 #include <linux/kernel.h> 11 - #include <linux/module.h> 12 10 #include <linux/cpufeature.h> 13 11 #include <linux/sizes.h> 14 12 #include <asm/simd.h> ··· 46 48 chacha_crypt_generic(state, dst, src, bytes, nrounds); 47 49 } 48 50 49 - void hchacha_block_arch(const struct chacha_state *state, 50 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 51 - { 52 - hchacha_block_generic(state, out, nrounds); 53 - } 54 - EXPORT_SYMBOL(hchacha_block_arch); 51 + #define hchacha_block_arch hchacha_block_generic /* not implemented yet */ 55 52 56 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 57 - unsigned int bytes, int nrounds) 53 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 54 + const u8 *src, unsigned int bytes, int nrounds) 58 55 { 59 56 if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE || 60 57 !crypto_simd_usable()) ··· 67 74 dst += todo; 68 75 } while (bytes); 69 76 } 70 - EXPORT_SYMBOL(chacha_crypt_arch); 71 77 72 - static int __init chacha_p10_init(void) 78 + #define chacha_mod_init_arch chacha_mod_init_arch 79 + static void chacha_mod_init_arch(void) 73 80 { 74 81 if (cpu_has_feature(CPU_FTR_ARCH_31)) 75 82 static_branch_enable(&have_p10); 76 - return 0; 77 83 } 78 - subsys_initcall(chacha_p10_init); 79 - 80 - static void __exit chacha_p10_exit(void) 81 - { 82 - } 83 - module_exit(chacha_p10_exit); 84 - 85 - MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)"); 86 - MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); 87 - MODULE_LICENSE("GPL v2");
-8
lib/crypto/riscv/Kconfig
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - config CRYPTO_CHACHA_RISCV64 4 - tristate 5 - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO 6 - default CRYPTO_LIB_CHACHA 7 - select CRYPTO_ARCH_HAVE_LIB_CHACHA 8 - select CRYPTO_LIB_CHACHA_GENERIC
-4
lib/crypto/riscv/Makefile
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o 4 - chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
+6 -24
lib/crypto/riscv/chacha-riscv64-glue.c lib/crypto/riscv/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0-only 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 /* 3 3 * ChaCha stream cipher (RISC-V optimized) 4 4 * ··· 8 8 9 9 #include <asm/simd.h> 10 10 #include <asm/vector.h> 11 - #include <crypto/chacha.h> 12 11 #include <crypto/internal/simd.h> 13 12 #include <linux/linkage.h> 14 - #include <linux/module.h> 15 13 16 14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); 17 15 18 16 asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out, 19 17 size_t nblocks, int nrounds); 20 18 21 - void hchacha_block_arch(const struct chacha_state *state, 22 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 23 - { 24 - hchacha_block_generic(state, out, nrounds); 25 - } 26 - EXPORT_SYMBOL(hchacha_block_arch); 19 + #define hchacha_block_arch hchacha_block_generic /* not implemented yet */ 27 20 28 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 29 - unsigned int bytes, int nrounds) 21 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 22 + const u8 *src, unsigned int bytes, int nrounds) 30 23 { 31 24 u8 block_buffer[CHACHA_BLOCK_SIZE]; 32 25 unsigned int full_blocks = bytes / CHACHA_BLOCK_SIZE; ··· 41 48 } 42 49 kernel_vector_end(); 43 50 } 44 - EXPORT_SYMBOL(chacha_crypt_arch); 45 51 46 - static int __init riscv64_chacha_mod_init(void) 52 + #define chacha_mod_init_arch chacha_mod_init_arch 53 + static void chacha_mod_init_arch(void) 47 54 { 48 55 if (riscv_isa_extension_available(NULL, ZVKB) && 49 56 riscv_vector_vlen() >= 128) 50 57 static_branch_enable(&use_zvkb); 51 - return 0; 52 58 } 53 - subsys_initcall(riscv64_chacha_mod_init); 54 - 55 - static void __exit riscv64_chacha_mod_exit(void) 56 - { 57 - } 58 - module_exit(riscv64_chacha_mod_exit); 59 - 60 - MODULE_DESCRIPTION("ChaCha stream cipher (RISC-V optimized)"); 61 - MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>"); 62 - MODULE_LICENSE("GPL");
-7
lib/crypto/s390/Kconfig
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - config CRYPTO_CHACHA_S390 4 - tristate 5 - default CRYPTO_LIB_CHACHA 6 - select CRYPTO_LIB_CHACHA_GENERIC 7 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-4
lib/crypto/s390/Makefile
··· 1 - # SPDX-License-Identifier: GPL-2.0-only 2 - 3 - obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o 4 - chacha_s390-y := chacha-glue.o chacha-s390.o
+4 -19
lib/crypto/s390/chacha-glue.c lib/crypto/s390/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 /* 3 3 * ChaCha stream cipher (s390 optimized) 4 4 * 5 5 * Copyright IBM Corp. 2021 6 6 */ 7 7 8 - #define KMSG_COMPONENT "chacha_s390" 9 - #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 - 11 - #include <crypto/chacha.h> 12 8 #include <linux/cpufeature.h> 13 9 #include <linux/export.h> 14 10 #include <linux/kernel.h> 15 - #include <linux/module.h> 16 11 #include <linux/sizes.h> 17 12 #include <asm/fpu.h> 18 13 #include "chacha-s390.h" 19 14 20 - void hchacha_block_arch(const struct chacha_state *state, 21 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 22 - { 23 - /* TODO: implement hchacha_block_arch() in assembly */ 24 - hchacha_block_generic(state, out, nrounds); 25 - } 26 - EXPORT_SYMBOL(hchacha_block_arch); 15 + #define hchacha_block_arch hchacha_block_generic /* not implemented yet */ 27 16 28 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 29 - unsigned int bytes, int nrounds) 17 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 18 + const u8 *src, unsigned int bytes, int nrounds) 30 19 { 31 20 /* s390 chacha20 implementation has 20 rounds hard-coded, 32 21 * it cannot handle a block of data or less, but otherwise ··· 34 45 CHACHA_BLOCK_SIZE; 35 46 } 36 47 } 37 - EXPORT_SYMBOL(chacha_crypt_arch); 38 - 39 - MODULE_DESCRIPTION("ChaCha stream cipher (s390 optimized)"); 40 - MODULE_LICENSE("GPL v2");
-7
lib/crypto/x86/Kconfig
··· 11 11 Architecture: x86_64 using: 12 12 - SSSE3 (Supplemental SSE3) 13 13 - AVX-512 (Advanced Vector Extensions-512) 14 - 15 - config CRYPTO_CHACHA20_X86_64 16 - tristate 17 - depends on 64BIT 18 - default CRYPTO_LIB_CHACHA 19 - select CRYPTO_LIB_CHACHA_GENERIC 20 - select CRYPTO_ARCH_HAVE_LIB_CHACHA
-3
lib/crypto/x86/Makefile
··· 2 2 3 3 obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += libblake2s-x86_64.o 4 4 libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o 5 - 6 - obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o 7 - chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o
+8 -22
lib/crypto/x86/chacha_glue.c lib/crypto/x86/chacha.h
··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 /* 3 3 * ChaCha and HChaCha functions (x86_64 optimized) 4 4 * ··· 6 6 */ 7 7 8 8 #include <asm/simd.h> 9 - #include <crypto/chacha.h> 10 9 #include <linux/jump_label.h> 11 10 #include <linux/kernel.h> 12 - #include <linux/module.h> 13 11 #include <linux/sizes.h> 14 12 15 13 asmlinkage void chacha_block_xor_ssse3(const struct chacha_state *state, ··· 124 126 } 125 127 } 126 128 127 - void hchacha_block_arch(const struct chacha_state *state, 128 - u32 out[HCHACHA_OUT_WORDS], int nrounds) 129 + static void hchacha_block_arch(const struct chacha_state *state, 130 + u32 out[HCHACHA_OUT_WORDS], int nrounds) 129 131 { 130 132 if (!static_branch_likely(&chacha_use_simd)) { 131 133 hchacha_block_generic(state, out, nrounds); ··· 135 137 kernel_fpu_end(); 136 138 } 137 139 } 138 - EXPORT_SYMBOL(hchacha_block_arch); 139 140 140 - void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 141 - unsigned int bytes, int nrounds) 141 + static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 142 + const u8 *src, unsigned int bytes, int nrounds) 142 143 { 143 144 if (!static_branch_likely(&chacha_use_simd) || 144 145 bytes <= CHACHA_BLOCK_SIZE) ··· 155 158 dst += todo; 156 159 } while (bytes); 157 160 } 158 - EXPORT_SYMBOL(chacha_crypt_arch); 159 161 160 - static int __init chacha_simd_mod_init(void) 162 + #define chacha_mod_init_arch chacha_mod_init_arch 163 + static void chacha_mod_init_arch(void) 161 164 { 162 165 if (!boot_cpu_has(X86_FEATURE_SSSE3)) 163 - return 0; 166 + return; 164 167 165 168 static_branch_enable(&chacha_use_simd); 166 169 ··· 173 176 boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */ 174 177 static_branch_enable(&chacha_use_avx512vl); 175 178 } 176 - return 0; 177 179 } 178 - subsys_initcall(chacha_simd_mod_init); 179 - 180 - static void __exit chacha_simd_mod_exit(void) 181 - { 182 - } 183 - module_exit(chacha_simd_mod_exit); 184 - 185 - MODULE_LICENSE("GPL"); 186 - MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); 187 - MODULE_DESCRIPTION("ChaCha and HChaCha functions (x86_64 optimized)");