tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: arm64/sha1: Migrate optimized code into library
Instead of exposing the arm64-optimized SHA-1 code via arm64-specific
crypto_shash algorithms, instead just implement the sha1_blocks()
library function. This is much simpler, it makes the SHA-1 library
functions be arm64-optimized, and it fixes the longstanding issue where
the arm64-optimized SHA-1 code was disabled by default. SHA-1 still
remains available through crypto_shash, but individual architectures no
longer need to handle it.
Remove support for SHA-1 finalization from assembly code, since the
library does not yet support architecture-specific overrides of the
finalization. (Support for that has been omitted for now, for
simplicity and because usually it isn't performance-critical.)
To match sha1_blocks(), change the type of the nblocks parameter and the
return value of __sha1_ce_transform() from int to size_t. Update the
assembly code accordingly.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250712232329.818226-9-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+51
-163
-1
arch/arm64/configs/defconfig
-1
arch/arm64/configs/defconfig
-11
arch/arm64/crypto/Kconfig
-11
arch/arm64/crypto/Kconfig
···
25
25
Architecture: arm64 using:
26
26
- NEON (Advanced SIMD) extensions
27
27
28
-
config CRYPTO_SHA1_ARM64_CE
29
-
tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)"
30
-
depends on KERNEL_MODE_NEON
31
-
select CRYPTO_HASH
32
-
select CRYPTO_SHA1
33
-
help
34
-
SHA-1 secure hash algorithm (FIPS 180)
35
-
36
-
Architecture: arm64 using:
37
-
- ARMv8 Crypto Extensions
38
-
39
28
config CRYPTO_SHA3_ARM64
40
29
tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)"
41
30
depends on KERNEL_MODE_NEON
-3
arch/arm64/crypto/Makefile
-3
arch/arm64/crypto/Makefile
+10
-30
arch/arm64/crypto/sha1-ce-core.S
lib/crypto/arm64/sha1-ce-core.S
+10
-30
arch/arm64/crypto/sha1-ce-core.S
lib/crypto/arm64/sha1-ce-core.S
···
62
62
.endm
63
63
64
64
/*
65
-
* int __sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src,
66
-
* int blocks)
65
+
* size_t __sha1_ce_transform(struct sha1_block_state *state,
66
+
* const u8 *data, size_t nblocks);
67
67
*/
68
68
SYM_FUNC_START(__sha1_ce_transform)
69
69
/* load round constants */
···
76
76
ld1 {dgav.4s}, [x0]
77
77
ldr dgb, [x0, #16]
78
78
79
-
/* load sha1_ce_state::finalize */
80
-
ldr_l w4, sha1_ce_offsetof_finalize, x4
81
-
ldr w4, [x0, x4]
82
-
83
79
/* load input */
84
80
0: ld1 {v8.4s-v11.4s}, [x1], #64
85
-
sub w2, w2, #1
81
+
sub x2, x2, #1
86
82
87
83
CPU_LE( rev32 v8.16b, v8.16b )
88
84
CPU_LE( rev32 v9.16b, v9.16b )
89
85
CPU_LE( rev32 v10.16b, v10.16b )
90
86
CPU_LE( rev32 v11.16b, v11.16b )
91
87
92
-
1: add t0.4s, v8.4s, k0.4s
88
+
add t0.4s, v8.4s, k0.4s
93
89
mov dg0v.16b, dgav.16b
94
90
95
91
add_update c, ev, k0, 8, 9, 10, 11, dgb
···
116
120
add dgbv.2s, dgbv.2s, dg1v.2s
117
121
add dgav.4s, dgav.4s, dg0v.4s
118
122
119
-
cbz w2, 2f
120
-
cond_yield 3f, x5, x6
121
-
b 0b
123
+
/* return early if voluntary preemption is needed */
124
+
cond_yield 1f, x5, x6
122
125
123
-
/*
124
-
* Final block: add padding and total bit count.
125
-
* Skip if the input size was not a round multiple of the block size,
126
-
* the padding is handled by the C code in that case.
127
-
*/
128
-
2: cbz x4, 3f
129
-
ldr_l w4, sha1_ce_offsetof_count, x4
130
-
ldr x4, [x0, x4]
131
-
movi v9.2d, #0
132
-
mov x8, #0x80000000
133
-
movi v10.2d, #0
134
-
ror x7, x4, #29 // ror(lsl(x4, 3), 32)
135
-
fmov d8, x8
136
-
mov x4, #0
137
-
mov v11.d[0], xzr
138
-
mov v11.d[1], x7
139
-
b 1b
126
+
/* handled all input blocks? */
127
+
cbnz x2, 0b
140
128
141
129
/* store new state */
142
-
3: st1 {dgav.4s}, [x0]
130
+
1: st1 {dgav.4s}, [x0]
143
131
str dgb, [x0, #16]
144
-
mov w0, w2
132
+
mov x0, x2
145
133
ret
146
134
SYM_FUNC_END(__sha1_ce_transform)
-118
arch/arm64/crypto/sha1-ce-glue.c
-118
arch/arm64/crypto/sha1-ce-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
4
-
*
5
-
* Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <asm/neon.h>
9
-
#include <asm/simd.h>
10
-
#include <crypto/internal/hash.h>
11
-
#include <crypto/internal/simd.h>
12
-
#include <crypto/sha1.h>
13
-
#include <crypto/sha1_base.h>
14
-
#include <linux/cpufeature.h>
15
-
#include <linux/kernel.h>
16
-
#include <linux/module.h>
17
-
#include <linux/string.h>
18
-
19
-
MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
20
-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
21
-
MODULE_LICENSE("GPL v2");
22
-
MODULE_ALIAS_CRYPTO("sha1");
23
-
24
-
struct sha1_ce_state {
25
-
struct sha1_state sst;
26
-
u32 finalize;
27
-
};
28
-
29
-
extern const u32 sha1_ce_offsetof_count;
30
-
extern const u32 sha1_ce_offsetof_finalize;
31
-
32
-
asmlinkage int __sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src,
33
-
int blocks);
34
-
35
-
static void sha1_ce_transform(struct sha1_state *sst, u8 const *src,
36
-
int blocks)
37
-
{
38
-
while (blocks) {
39
-
int rem;
40
-
41
-
kernel_neon_begin();
42
-
rem = __sha1_ce_transform(container_of(sst,
43
-
struct sha1_ce_state,
44
-
sst), src, blocks);
45
-
kernel_neon_end();
46
-
src += (blocks - rem) * SHA1_BLOCK_SIZE;
47
-
blocks = rem;
48
-
}
49
-
}
50
-
51
-
const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count);
52
-
const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize);
53
-
54
-
static int sha1_ce_update(struct shash_desc *desc, const u8 *data,
55
-
unsigned int len)
56
-
{
57
-
struct sha1_ce_state *sctx = shash_desc_ctx(desc);
58
-
59
-
sctx->finalize = 0;
60
-
return sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform);
61
-
}
62
-
63
-
static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
64
-
unsigned int len, u8 *out)
65
-
{
66
-
struct sha1_ce_state *sctx = shash_desc_ctx(desc);
67
-
bool finalized = false;
68
-
69
-
/*
70
-
* Allow the asm code to perform the finalization if there is no
71
-
* partial data and the input is a round multiple of the block size.
72
-
*/
73
-
if (len >= SHA1_BLOCK_SIZE) {
74
-
unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE);
75
-
76
-
finalized = !remain;
77
-
sctx->finalize = finalized;
78
-
sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform);
79
-
data += len - remain;
80
-
len = remain;
81
-
}
82
-
if (!finalized) {
83
-
sctx->finalize = 0;
84
-
sha1_base_do_finup(desc, data, len, sha1_ce_transform);
85
-
}
86
-
return sha1_base_finish(desc, out);
87
-
}
88
-
89
-
static struct shash_alg alg = {
90
-
.init = sha1_base_init,
91
-
.update = sha1_ce_update,
92
-
.finup = sha1_ce_finup,
93
-
.descsize = sizeof(struct sha1_ce_state),
94
-
.statesize = SHA1_STATE_SIZE,
95
-
.digestsize = SHA1_DIGEST_SIZE,
96
-
.base = {
97
-
.cra_name = "sha1",
98
-
.cra_driver_name = "sha1-ce",
99
-
.cra_priority = 200,
100
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
101
-
CRYPTO_AHASH_ALG_FINUP_MAX,
102
-
.cra_blocksize = SHA1_BLOCK_SIZE,
103
-
.cra_module = THIS_MODULE,
104
-
}
105
-
};
106
-
107
-
static int __init sha1_ce_mod_init(void)
108
-
{
109
-
return crypto_register_shash(&alg);
110
-
}
111
-
112
-
static void __exit sha1_ce_mod_fini(void)
113
-
{
114
-
crypto_unregister_shash(&alg);
115
-
}
116
-
117
-
module_cpu_feature_match(SHA1, sha1_ce_mod_init);
118
-
module_exit(sha1_ce_mod_fini);
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+1
lib/crypto/Makefile
+1
lib/crypto/Makefile
+39
lib/crypto/arm64/sha1.h
+39
lib/crypto/arm64/sha1.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* SHA-1 optimized for ARM64
4
+
*
5
+
* Copyright 2025 Google LLC
6
+
*/
7
+
#include <asm/neon.h>
8
+
#include <asm/simd.h>
9
+
#include <linux/cpufeature.h>
10
+
11
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
12
+
13
+
asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state,
14
+
const u8 *data, size_t nblocks);
15
+
16
+
static void sha1_blocks(struct sha1_block_state *state,
17
+
const u8 *data, size_t nblocks)
18
+
{
19
+
if (static_branch_likely(&have_ce) && likely(may_use_simd())) {
20
+
do {
21
+
size_t rem;
22
+
23
+
kernel_neon_begin();
24
+
rem = __sha1_ce_transform(state, data, nblocks);
25
+
kernel_neon_end();
26
+
data += (nblocks - rem) * SHA1_BLOCK_SIZE;
27
+
nblocks = rem;
28
+
} while (nblocks);
29
+
} else {
30
+
sha1_blocks_generic(state, data, nblocks);
31
+
}
32
+
}
33
+
34
+
#define sha1_mod_init_arch sha1_mod_init_arch
35
+
static inline void sha1_mod_init_arch(void)
36
+
{
37
+
if (cpu_have_named_feature(SHA1))
38
+
static_branch_enable(&have_ce);
39
+
}