Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Cryptographic API.
4 *
5 * Support for VIA PadLock hardware crypto engine.
6 *
7 * Copyright (c) 2006 Michal Ludvig <michal@logix.cz>
8 */
9
10#include <asm/cpu_device_id.h>
11#include <crypto/internal/hash.h>
12#include <crypto/padlock.h>
13#include <crypto/sha1.h>
14#include <crypto/sha2.h>
15#include <linux/cpufeature.h>
16#include <linux/err.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19
20#define PADLOCK_SHA_DESCSIZE (128 + ((PADLOCK_ALIGNMENT - 1) & \
21 ~(CRYPTO_MINALIGN - 1)))
22
23struct padlock_sha_ctx {
24 struct crypto_ahash *fallback;
25};
26
27static inline void *padlock_shash_desc_ctx(struct shash_desc *desc)
28{
29 return PTR_ALIGN(shash_desc_ctx(desc), PADLOCK_ALIGNMENT);
30}
31
32static int padlock_sha1_init(struct shash_desc *desc)
33{
34 struct sha1_state *sctx = padlock_shash_desc_ctx(desc);
35
36 *sctx = (struct sha1_state){
37 .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
38 };
39
40 return 0;
41}
42
43static int padlock_sha256_init(struct shash_desc *desc)
44{
45 struct crypto_sha256_state *sctx = padlock_shash_desc_ctx(desc);
46
47 sha256_block_init(sctx);
48 return 0;
49}
50
51static int padlock_sha_update(struct shash_desc *desc,
52 const u8 *data, unsigned int length)
53{
54 u8 *state = padlock_shash_desc_ctx(desc);
55 struct crypto_shash *tfm = desc->tfm;
56 int err, remain;
57
58 remain = length - round_down(length, crypto_shash_blocksize(tfm));
59 {
60 struct padlock_sha_ctx *ctx = crypto_shash_ctx(tfm);
61 HASH_REQUEST_ON_STACK(req, ctx->fallback);
62
63 ahash_request_set_callback(req, 0, NULL, NULL);
64 ahash_request_set_virt(req, data, NULL, length - remain);
65 err = crypto_ahash_import_core(req, state) ?:
66 crypto_ahash_update(req) ?:
67 crypto_ahash_export_core(req, state);
68 HASH_REQUEST_ZERO(req);
69 }
70
71 return err ?: remain;
72}
73
74static int padlock_sha_export(struct shash_desc *desc, void *out)
75{
76 memcpy(out, padlock_shash_desc_ctx(desc),
77 crypto_shash_coresize(desc->tfm));
78 return 0;
79}
80
81static int padlock_sha_import(struct shash_desc *desc, const void *in)
82{
83 unsigned int bs = crypto_shash_blocksize(desc->tfm);
84 unsigned int ss = crypto_shash_coresize(desc->tfm);
85 u64 *state = padlock_shash_desc_ctx(desc);
86
87 memcpy(state, in, ss);
88
89 /* Stop evil imports from generating a fault. */
90 state[ss / 8 - 1] &= ~(bs - 1);
91
92 return 0;
93}
94
95static inline void padlock_output_block(uint32_t *src,
96 uint32_t *dst, size_t count)
97{
98 while (count--)
99 *dst++ = swab32(*src++);
100}
101
102static int padlock_sha_finup(struct shash_desc *desc, const u8 *in,
103 unsigned int count, u8 *out)
104{
105 struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
106 HASH_REQUEST_ON_STACK(req, ctx->fallback);
107
108 ahash_request_set_callback(req, 0, NULL, NULL);
109 ahash_request_set_virt(req, in, out, count);
110 return crypto_ahash_import_core(req, padlock_shash_desc_ctx(desc)) ?:
111 crypto_ahash_finup(req);
112}
113
114static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in,
115 unsigned int count, u8 *out)
116{
117 /* We can't store directly to *out as it may be unaligned. */
118 /* BTW Don't reduce the buffer size below 128 Bytes!
119 * PadLock microcode needs it that big. */
120 struct sha1_state *state = padlock_shash_desc_ctx(desc);
121 u64 start = state->count;
122
123 if (start + count > ULONG_MAX)
124 return padlock_sha_finup(desc, in, count, out);
125
126 asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
127 : \
128 : "c"((unsigned long)start + count), \
129 "a"((unsigned long)start), \
130 "S"(in), "D"(state));
131
132 padlock_output_block(state->state, (uint32_t *)out, 5);
133 return 0;
134}
135
136static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in,
137 unsigned int count, u8 *out)
138{
139 /* We can't store directly to *out as it may be unaligned. */
140 /* BTW Don't reduce the buffer size below 128 Bytes!
141 * PadLock microcode needs it that big. */
142 struct sha256_state *state = padlock_shash_desc_ctx(desc);
143 u64 start = state->count;
144
145 if (start + count > ULONG_MAX)
146 return padlock_sha_finup(desc, in, count, out);
147
148 asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
149 : \
150 : "c"((unsigned long)start + count), \
151 "a"((unsigned long)start), \
152 "S"(in), "D"(state));
153
154 padlock_output_block(state->state, (uint32_t *)out, 8);
155 return 0;
156}
157
158static int padlock_init_tfm(struct crypto_shash *hash)
159{
160 const char *fallback_driver_name = crypto_shash_alg_name(hash);
161 struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash);
162 struct crypto_ahash *fallback_tfm;
163
164 /* Allocate a fallback and abort if it failed. */
165 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
166 CRYPTO_ALG_NEED_FALLBACK |
167 CRYPTO_ALG_ASYNC);
168 if (IS_ERR(fallback_tfm)) {
169 printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
170 fallback_driver_name);
171 return PTR_ERR(fallback_tfm);
172 }
173
174 if (crypto_shash_statesize(hash) !=
175 crypto_ahash_statesize(fallback_tfm)) {
176 crypto_free_ahash(fallback_tfm);
177 return -EINVAL;
178 }
179
180 ctx->fallback = fallback_tfm;
181
182 return 0;
183}
184
185static void padlock_exit_tfm(struct crypto_shash *hash)
186{
187 struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash);
188
189 crypto_free_ahash(ctx->fallback);
190}
191
192static struct shash_alg sha1_alg = {
193 .digestsize = SHA1_DIGEST_SIZE,
194 .init = padlock_sha1_init,
195 .update = padlock_sha_update,
196 .finup = padlock_sha1_finup,
197 .export = padlock_sha_export,
198 .import = padlock_sha_import,
199 .init_tfm = padlock_init_tfm,
200 .exit_tfm = padlock_exit_tfm,
201 .descsize = PADLOCK_SHA_DESCSIZE,
202 .statesize = SHA1_STATE_SIZE,
203 .base = {
204 .cra_name = "sha1",
205 .cra_driver_name = "sha1-padlock",
206 .cra_priority = PADLOCK_CRA_PRIORITY,
207 .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
208 CRYPTO_AHASH_ALG_BLOCK_ONLY |
209 CRYPTO_AHASH_ALG_FINUP_MAX,
210 .cra_blocksize = SHA1_BLOCK_SIZE,
211 .cra_ctxsize = sizeof(struct padlock_sha_ctx),
212 .cra_module = THIS_MODULE,
213 }
214};
215
216static struct shash_alg sha256_alg = {
217 .digestsize = SHA256_DIGEST_SIZE,
218 .init = padlock_sha256_init,
219 .update = padlock_sha_update,
220 .finup = padlock_sha256_finup,
221 .init_tfm = padlock_init_tfm,
222 .export = padlock_sha_export,
223 .import = padlock_sha_import,
224 .exit_tfm = padlock_exit_tfm,
225 .descsize = PADLOCK_SHA_DESCSIZE,
226 .statesize = sizeof(struct crypto_sha256_state),
227 .base = {
228 .cra_name = "sha256",
229 .cra_driver_name = "sha256-padlock",
230 .cra_priority = PADLOCK_CRA_PRIORITY,
231 .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
232 CRYPTO_AHASH_ALG_BLOCK_ONLY |
233 CRYPTO_AHASH_ALG_FINUP_MAX,
234 .cra_blocksize = SHA256_BLOCK_SIZE,
235 .cra_ctxsize = sizeof(struct padlock_sha_ctx),
236 .cra_module = THIS_MODULE,
237 }
238};
239
240/* Add two shash_alg instance for hardware-implemented *
241* multiple-parts hash supported by VIA Nano Processor.*/
242
243static int padlock_sha1_update_nano(struct shash_desc *desc,
244 const u8 *src, unsigned int len)
245{
246 /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
247 struct sha1_state *state = padlock_shash_desc_ctx(desc);
248 int blocks = len / SHA1_BLOCK_SIZE;
249
250 len -= blocks * SHA1_BLOCK_SIZE;
251 state->count += blocks * SHA1_BLOCK_SIZE;
252
253 /* Process the left bytes from the input data */
254 asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
255 : "+S"(src), "+D"(state)
256 : "a"((long)-1),
257 "c"((unsigned long)blocks));
258 return len;
259}
260
261static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *src,
262 unsigned int len)
263{
264 /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
265 struct crypto_sha256_state *state = padlock_shash_desc_ctx(desc);
266 int blocks = len / SHA256_BLOCK_SIZE;
267
268 len -= blocks * SHA256_BLOCK_SIZE;
269 state->count += blocks * SHA256_BLOCK_SIZE;
270
271 /* Process the left bytes from input data*/
272 asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
273 : "+S"(src), "+D"(state)
274 : "a"((long)-1),
275 "c"((unsigned long)blocks));
276 return len;
277}
278
279static struct shash_alg sha1_alg_nano = {
280 .digestsize = SHA1_DIGEST_SIZE,
281 .init = padlock_sha1_init,
282 .update = padlock_sha1_update_nano,
283 .finup = padlock_sha1_finup,
284 .export = padlock_sha_export,
285 .import = padlock_sha_import,
286 .descsize = PADLOCK_SHA_DESCSIZE,
287 .statesize = SHA1_STATE_SIZE,
288 .base = {
289 .cra_name = "sha1",
290 .cra_driver_name = "sha1-padlock-nano",
291 .cra_priority = PADLOCK_CRA_PRIORITY,
292 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
293 CRYPTO_AHASH_ALG_FINUP_MAX,
294 .cra_blocksize = SHA1_BLOCK_SIZE,
295 .cra_module = THIS_MODULE,
296 }
297};
298
299static struct shash_alg sha256_alg_nano = {
300 .digestsize = SHA256_DIGEST_SIZE,
301 .init = padlock_sha256_init,
302 .update = padlock_sha256_update_nano,
303 .finup = padlock_sha256_finup,
304 .export = padlock_sha_export,
305 .import = padlock_sha_import,
306 .descsize = PADLOCK_SHA_DESCSIZE,
307 .statesize = sizeof(struct crypto_sha256_state),
308 .base = {
309 .cra_name = "sha256",
310 .cra_driver_name = "sha256-padlock-nano",
311 .cra_priority = PADLOCK_CRA_PRIORITY,
312 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
313 CRYPTO_AHASH_ALG_FINUP_MAX,
314 .cra_blocksize = SHA256_BLOCK_SIZE,
315 .cra_module = THIS_MODULE,
316 }
317};
318
319static const struct x86_cpu_id padlock_sha_ids[] = {
320 X86_MATCH_FEATURE(X86_FEATURE_PHE, NULL),
321 {}
322};
323MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids);
324
325static int __init padlock_init(void)
326{
327 int rc = -ENODEV;
328 struct cpuinfo_x86 *c = &cpu_data(0);
329 struct shash_alg *sha1;
330 struct shash_alg *sha256;
331
332 if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN))
333 return -ENODEV;
334
335 /* Register the newly added algorithm module if on *
336 * VIA Nano processor, or else just do as before */
337 if (c->x86_model < 0x0f) {
338 sha1 = &sha1_alg;
339 sha256 = &sha256_alg;
340 } else {
341 sha1 = &sha1_alg_nano;
342 sha256 = &sha256_alg_nano;
343 }
344
345 rc = crypto_register_shash(sha1);
346 if (rc)
347 goto out;
348
349 rc = crypto_register_shash(sha256);
350 if (rc)
351 goto out_unreg1;
352
353 printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
354
355 return 0;
356
357out_unreg1:
358 crypto_unregister_shash(sha1);
359
360out:
361 printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
362 return rc;
363}
364
365static void __exit padlock_fini(void)
366{
367 struct cpuinfo_x86 *c = &cpu_data(0);
368
369 if (c->x86_model >= 0x0f) {
370 crypto_unregister_shash(&sha1_alg_nano);
371 crypto_unregister_shash(&sha256_alg_nano);
372 } else {
373 crypto_unregister_shash(&sha1_alg);
374 crypto_unregister_shash(&sha256_alg);
375 }
376}
377
378module_init(padlock_init);
379module_exit(padlock_fini);
380
381MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
382MODULE_LICENSE("GPL");
383MODULE_AUTHOR("Michal Ludvig");
384
385MODULE_ALIAS_CRYPTO("sha1-all");
386MODULE_ALIAS_CRYPTO("sha256-all");
387MODULE_ALIAS_CRYPTO("sha1-padlock");
388MODULE_ALIAS_CRYPTO("sha256-padlock");