tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: poly1305-p10 - Glue code for optmized Poly1305 implementation for ppc64le
Signed-off-by: Danny Tsen <dtsen@linux.ibm.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Danny Tsen
and committed by
Herbert Xu
ba8f8624
09ef057b
+186
+186
arch/powerpc/crypto/poly1305-p10-glue.c
+186
arch/powerpc/crypto/poly1305-p10-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Poly1305 authenticator algorithm, RFC7539.
4
+
*
5
+
* Copyright 2023- IBM Corp. All rights reserved.
6
+
*/
7
+
8
+
#include <crypto/algapi.h>
9
+
#include <linux/crypto.h>
10
+
#include <linux/kernel.h>
11
+
#include <linux/module.h>
12
+
#include <linux/jump_label.h>
13
+
#include <crypto/internal/hash.h>
14
+
#include <crypto/internal/poly1305.h>
15
+
#include <crypto/internal/simd.h>
16
+
#include <linux/cpufeature.h>
17
+
#include <asm/unaligned.h>
18
+
#include <asm/simd.h>
19
+
#include <asm/switch_to.h>
20
+
21
+
asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
22
+
asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
23
+
asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
24
+
25
+
static void vsx_begin(void)
26
+
{
27
+
preempt_disable();
28
+
enable_kernel_vsx();
29
+
}
30
+
31
+
static void vsx_end(void)
32
+
{
33
+
disable_kernel_vsx();
34
+
preempt_enable();
35
+
}
36
+
37
+
static int crypto_poly1305_p10_init(struct shash_desc *desc)
38
+
{
39
+
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
40
+
41
+
poly1305_core_init(&dctx->h);
42
+
dctx->buflen = 0;
43
+
dctx->rset = 0;
44
+
dctx->sset = false;
45
+
46
+
return 0;
47
+
}
48
+
49
+
static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
50
+
const u8 *inp, unsigned int len)
51
+
{
52
+
unsigned int acc = 0;
53
+
54
+
if (unlikely(!dctx->sset)) {
55
+
if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {
56
+
struct poly1305_core_key *key = &dctx->core_r;
57
+
58
+
key->key.r64[0] = get_unaligned_le64(&inp[0]);
59
+
key->key.r64[1] = get_unaligned_le64(&inp[8]);
60
+
inp += POLY1305_BLOCK_SIZE;
61
+
len -= POLY1305_BLOCK_SIZE;
62
+
acc += POLY1305_BLOCK_SIZE;
63
+
dctx->rset = 1;
64
+
}
65
+
if (len >= POLY1305_BLOCK_SIZE) {
66
+
dctx->s[0] = get_unaligned_le32(&inp[0]);
67
+
dctx->s[1] = get_unaligned_le32(&inp[4]);
68
+
dctx->s[2] = get_unaligned_le32(&inp[8]);
69
+
dctx->s[3] = get_unaligned_le32(&inp[12]);
70
+
acc += POLY1305_BLOCK_SIZE;
71
+
dctx->sset = true;
72
+
}
73
+
}
74
+
return acc;
75
+
}
76
+
77
+
static int crypto_poly1305_p10_update(struct shash_desc *desc,
78
+
const u8 *src, unsigned int srclen)
79
+
{
80
+
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
81
+
unsigned int bytes, used;
82
+
83
+
if (unlikely(dctx->buflen)) {
84
+
bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
85
+
memcpy(dctx->buf + dctx->buflen, src, bytes);
86
+
src += bytes;
87
+
srclen -= bytes;
88
+
dctx->buflen += bytes;
89
+
90
+
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
91
+
if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf,
92
+
POLY1305_BLOCK_SIZE))) {
93
+
vsx_begin();
94
+
poly1305_64s(&dctx->h, dctx->buf,
95
+
POLY1305_BLOCK_SIZE, 1);
96
+
vsx_end();
97
+
}
98
+
dctx->buflen = 0;
99
+
}
100
+
}
101
+
102
+
if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
103
+
bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
104
+
used = crypto_poly1305_setdctxkey(dctx, src, bytes);
105
+
if (likely(used)) {
106
+
srclen -= used;
107
+
src += used;
108
+
}
109
+
if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
110
+
vsx_begin();
111
+
poly1305_p10le_4blocks(&dctx->h, src, srclen);
112
+
vsx_end();
113
+
src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
114
+
srclen %= POLY1305_BLOCK_SIZE * 4;
115
+
}
116
+
while (srclen >= POLY1305_BLOCK_SIZE) {
117
+
vsx_begin();
118
+
poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1);
119
+
vsx_end();
120
+
srclen -= POLY1305_BLOCK_SIZE;
121
+
src += POLY1305_BLOCK_SIZE;
122
+
}
123
+
}
124
+
125
+
if (unlikely(srclen)) {
126
+
dctx->buflen = srclen;
127
+
memcpy(dctx->buf, src, srclen);
128
+
}
129
+
130
+
return 0;
131
+
}
132
+
133
+
static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst)
134
+
{
135
+
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
136
+
137
+
if (unlikely(!dctx->sset))
138
+
return -ENOKEY;
139
+
140
+
if ((dctx->buflen)) {
141
+
dctx->buf[dctx->buflen++] = 1;
142
+
memset(dctx->buf + dctx->buflen, 0,
143
+
POLY1305_BLOCK_SIZE - dctx->buflen);
144
+
vsx_begin();
145
+
poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
146
+
vsx_end();
147
+
dctx->buflen = 0;
148
+
}
149
+
150
+
poly1305_emit_64(&dctx->h, &dctx->s, dst);
151
+
return 0;
152
+
}
153
+
154
+
static struct shash_alg poly1305_alg = {
155
+
.digestsize = POLY1305_DIGEST_SIZE,
156
+
.init = crypto_poly1305_p10_init,
157
+
.update = crypto_poly1305_p10_update,
158
+
.final = crypto_poly1305_p10_final,
159
+
.descsize = sizeof(struct poly1305_desc_ctx),
160
+
.base = {
161
+
.cra_name = "poly1305",
162
+
.cra_driver_name = "poly1305-p10",
163
+
.cra_priority = 300,
164
+
.cra_blocksize = POLY1305_BLOCK_SIZE,
165
+
.cra_module = THIS_MODULE,
166
+
},
167
+
};
168
+
169
+
static int __init poly1305_p10_init(void)
170
+
{
171
+
return crypto_register_shash(&poly1305_alg);
172
+
}
173
+
174
+
static void __exit poly1305_p10_exit(void)
175
+
{
176
+
crypto_unregister_shash(&poly1305_alg);
177
+
}
178
+
179
+
module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init);
180
+
module_exit(poly1305_p10_exit);
181
+
182
+
MODULE_LICENSE("GPL");
183
+
MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
184
+
MODULE_DESCRIPTION("Optimized Poly1305 for P10");
185
+
MODULE_ALIAS_CRYPTO("poly1305");
186
+
MODULE_ALIAS_CRYPTO("poly1305-p10");