tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <crypto/algapi.h>
22#include <crypto/chacha20.h>
23#include <crypto/internal/skcipher.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26
27#include <asm/hwcap.h>
28#include <asm/neon.h>
29#include <asm/simd.h>
30
31asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
32asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
33
34static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
35 unsigned int bytes)
36{
37 u8 buf[CHACHA20_BLOCK_SIZE];
38
39 while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
40 chacha20_4block_xor_neon(state, dst, src);
41 bytes -= CHACHA20_BLOCK_SIZE * 4;
42 src += CHACHA20_BLOCK_SIZE * 4;
43 dst += CHACHA20_BLOCK_SIZE * 4;
44 state[12] += 4;
45 }
46 while (bytes >= CHACHA20_BLOCK_SIZE) {
47 chacha20_block_xor_neon(state, dst, src);
48 bytes -= CHACHA20_BLOCK_SIZE;
49 src += CHACHA20_BLOCK_SIZE;
50 dst += CHACHA20_BLOCK_SIZE;
51 state[12]++;
52 }
53 if (bytes) {
54 memcpy(buf, src, bytes);
55 chacha20_block_xor_neon(state, buf, buf);
56 memcpy(dst, buf, bytes);
57 }
58}
59
60static int chacha20_neon(struct skcipher_request *req)
61{
62 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
63 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
64 struct skcipher_walk walk;
65 u32 state[16];
66 int err;
67
68 if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
69 return crypto_chacha20_crypt(req);
70
71 err = skcipher_walk_virt(&walk, req, true);
72
73 crypto_chacha20_init(state, ctx, walk.iv);
74
75 kernel_neon_begin();
76 while (walk.nbytes > 0) {
77 unsigned int nbytes = walk.nbytes;
78
79 if (nbytes < walk.total)
80 nbytes = round_down(nbytes, walk.stride);
81
82 chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
83 nbytes);
84 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
85 }
86 kernel_neon_end();
87
88 return err;
89}
90
91static struct skcipher_alg alg = {
92 .base.cra_name = "chacha20",
93 .base.cra_driver_name = "chacha20-neon",
94 .base.cra_priority = 300,
95 .base.cra_blocksize = 1,
96 .base.cra_ctxsize = sizeof(struct chacha20_ctx),
97 .base.cra_module = THIS_MODULE,
98
99 .min_keysize = CHACHA20_KEY_SIZE,
100 .max_keysize = CHACHA20_KEY_SIZE,
101 .ivsize = CHACHA20_IV_SIZE,
102 .chunksize = CHACHA20_BLOCK_SIZE,
103 .walksize = 4 * CHACHA20_BLOCK_SIZE,
104 .setkey = crypto_chacha20_setkey,
105 .encrypt = chacha20_neon,
106 .decrypt = chacha20_neon,
107};
108
109static int __init chacha20_simd_mod_init(void)
110{
111 if (!(elf_hwcap & HWCAP_NEON))
112 return -ENODEV;
113
114 return crypto_register_skcipher(&alg);
115}
116
117static void __exit chacha20_simd_mod_fini(void)
118{
119 crypto_unregister_skcipher(&alg);
120}
121
122module_init(chacha20_simd_mod_init);
123module_exit(chacha20_simd_mod_fini);
124
125MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
126MODULE_LICENSE("GPL v2");
127MODULE_ALIAS_CRYPTO("chacha20");