Gitiles
Code Review Sign In
192.168.1.100 / T800 / ae7cf9c1566c697c2ba616a120c7ff3e90d20dfd / . / src / kernel / linux / v4.19 / arch / arm64 / crypto / chacha20-neon-glue.c
blob: 96e0cfb8c3f5b3d88f51c9108b566158d4bbb76e [file] [log] [blame]
xjb04a4022021-11-25 15:01:52 +0800[diff] [blame]1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
3 *
4 * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <crypto/algapi.h>
22#include <crypto/chacha.h>
23#include <crypto/internal/skcipher.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26
27#include <asm/hwcap.h>
28#include <asm/neon.h>
29#include <asm/simd.h>
30
31asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
32asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
33
34static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
35 unsigned int bytes)
36{
37 u8 buf[CHACHA_BLOCK_SIZE];
38
39 while (bytes >= CHACHA_BLOCK_SIZE * 4) {
40 kernel_neon_begin();
41 chacha20_4block_xor_neon(state, dst, src);
42 kernel_neon_end();
43 bytes -= CHACHA_BLOCK_SIZE * 4;
44 src += CHACHA_BLOCK_SIZE * 4;
45 dst += CHACHA_BLOCK_SIZE * 4;
46 state[12] += 4;
47 }
48
49 if (!bytes)
50 return;
51
52 kernel_neon_begin();
53 while (bytes >= CHACHA_BLOCK_SIZE) {
54 chacha20_block_xor_neon(state, dst, src);
55 bytes -= CHACHA_BLOCK_SIZE;
56 src += CHACHA_BLOCK_SIZE;
57 dst += CHACHA_BLOCK_SIZE;
58 state[12]++;
59 }
60 if (bytes) {
61 memcpy(buf, src, bytes);
62 chacha20_block_xor_neon(state, buf, buf);
63 memcpy(dst, buf, bytes);
64 }
65 kernel_neon_end();
66}
67
68static int chacha20_neon(struct skcipher_request *req)
69{
70 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
71 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
72 struct skcipher_walk walk;
73 u32 state[16];
74 int err;
75
76 if (!may_use_simd() || req->cryptlen <= CHACHA_BLOCK_SIZE)
77 return crypto_chacha_crypt(req);
78
79 err = skcipher_walk_virt(&walk, req, false);
80
81 crypto_chacha_init(state, ctx, walk.iv);
82
83 while (walk.nbytes > 0) {
84 unsigned int nbytes = walk.nbytes;
85
86 if (nbytes < walk.total)
87 nbytes = round_down(nbytes, walk.stride);
88
89 chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
90 nbytes);
91 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
92 }
93
94 return err;
95}
96
97static struct skcipher_alg alg = {
98 .base.cra_name = "chacha20",
99 .base.cra_driver_name = "chacha20-neon",
100 .base.cra_priority = 300,
101 .base.cra_blocksize = 1,
102 .base.cra_ctxsize = sizeof(struct chacha_ctx),
103 .base.cra_module = THIS_MODULE,
104
105 .min_keysize = CHACHA_KEY_SIZE,
106 .max_keysize = CHACHA_KEY_SIZE,
107 .ivsize = CHACHA_IV_SIZE,
108 .chunksize = CHACHA_BLOCK_SIZE,
109 .walksize = 4 * CHACHA_BLOCK_SIZE,
110 .setkey = crypto_chacha20_setkey,
111 .encrypt = chacha20_neon,
112 .decrypt = chacha20_neon,
113};
114
115static int __init chacha20_simd_mod_init(void)
116{
117 if (!(elf_hwcap & HWCAP_ASIMD))
118 return -ENODEV;
119
120 return crypto_register_skcipher(&alg);
121}
122
123static void __exit chacha20_simd_mod_fini(void)
124{
125 crypto_unregister_skcipher(&alg);
126}
127
128module_init(chacha20_simd_mod_init);
129module_exit(chacha20_simd_mod_fini);
130
131MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
132MODULE_LICENSE("GPL v2");
133MODULE_ALIAS_CRYPTO("chacha20");