1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * SM4 Cipher Algorithm, using ARMv8 NEON
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
7 * Copyright (C) 2022, Alibaba Group.
8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
11 #include <linux/module.h>
12 #include <linux/crypto.h>
13 #include <linux/kernel.h>
14 #include <linux/cpufeature.h>
17 #include <crypto/internal/simd.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/sm4.h>
21 asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src,
22 unsigned int nblocks);
23 asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src,
24 u8 *iv, unsigned int nblocks);
25 asmlinkage void sm4_neon_cfb_dec(const u32 *rkey_enc, u8 *dst, const u8 *src,
26 u8 *iv, unsigned int nblocks);
27 asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src,
28 u8 *iv, unsigned int nblocks);
30 static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
33 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
35 return sm4_expandkey(ctx, key, key_len);
38 static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
40 struct skcipher_walk walk;
44 err = skcipher_walk_virt(&walk, req, false);
46 while ((nbytes = walk.nbytes) > 0) {
47 const u8 *src = walk.src.virt.addr;
48 u8 *dst = walk.dst.virt.addr;
51 nblocks = nbytes / SM4_BLOCK_SIZE;
55 sm4_neon_crypt(rkey, dst, src, nblocks);
60 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
66 static int sm4_ecb_encrypt(struct skcipher_request *req)
68 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
69 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
71 return sm4_ecb_do_crypt(req, ctx->rkey_enc);
74 static int sm4_ecb_decrypt(struct skcipher_request *req)
76 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
77 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
79 return sm4_ecb_do_crypt(req, ctx->rkey_dec);
82 static int sm4_cbc_encrypt(struct skcipher_request *req)
84 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
85 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
86 struct skcipher_walk walk;
90 err = skcipher_walk_virt(&walk, req, false);
92 while ((nbytes = walk.nbytes) > 0) {
93 const u8 *iv = walk.iv;
94 const u8 *src = walk.src.virt.addr;
95 u8 *dst = walk.dst.virt.addr;
97 while (nbytes >= SM4_BLOCK_SIZE) {
98 crypto_xor_cpy(dst, src, iv, SM4_BLOCK_SIZE);
99 sm4_crypt_block(ctx->rkey_enc, dst, dst);
101 src += SM4_BLOCK_SIZE;
102 dst += SM4_BLOCK_SIZE;
103 nbytes -= SM4_BLOCK_SIZE;
106 memcpy(walk.iv, iv, SM4_BLOCK_SIZE);
108 err = skcipher_walk_done(&walk, nbytes);
114 static int sm4_cbc_decrypt(struct skcipher_request *req)
116 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
117 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
118 struct skcipher_walk walk;
122 err = skcipher_walk_virt(&walk, req, false);
124 while ((nbytes = walk.nbytes) > 0) {
125 const u8 *src = walk.src.virt.addr;
126 u8 *dst = walk.dst.virt.addr;
127 unsigned int nblocks;
129 nblocks = nbytes / SM4_BLOCK_SIZE;
133 sm4_neon_cbc_dec(ctx->rkey_dec, dst, src,
139 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
145 static int sm4_cfb_encrypt(struct skcipher_request *req)
147 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
148 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
149 struct skcipher_walk walk;
153 err = skcipher_walk_virt(&walk, req, false);
155 while ((nbytes = walk.nbytes) > 0) {
156 u8 keystream[SM4_BLOCK_SIZE];
157 const u8 *iv = walk.iv;
158 const u8 *src = walk.src.virt.addr;
159 u8 *dst = walk.dst.virt.addr;
161 while (nbytes >= SM4_BLOCK_SIZE) {
162 sm4_crypt_block(ctx->rkey_enc, keystream, iv);
163 crypto_xor_cpy(dst, src, keystream, SM4_BLOCK_SIZE);
165 src += SM4_BLOCK_SIZE;
166 dst += SM4_BLOCK_SIZE;
167 nbytes -= SM4_BLOCK_SIZE;
170 memcpy(walk.iv, iv, SM4_BLOCK_SIZE);
173 if (walk.nbytes == walk.total && nbytes > 0) {
174 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv);
175 crypto_xor_cpy(dst, src, keystream, nbytes);
179 err = skcipher_walk_done(&walk, nbytes);
185 static int sm4_cfb_decrypt(struct skcipher_request *req)
187 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
188 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
189 struct skcipher_walk walk;
193 err = skcipher_walk_virt(&walk, req, false);
195 while ((nbytes = walk.nbytes) > 0) {
196 const u8 *src = walk.src.virt.addr;
197 u8 *dst = walk.dst.virt.addr;
198 unsigned int nblocks;
200 nblocks = nbytes / SM4_BLOCK_SIZE;
204 sm4_neon_cfb_dec(ctx->rkey_enc, dst, src,
209 dst += nblocks * SM4_BLOCK_SIZE;
210 src += nblocks * SM4_BLOCK_SIZE;
211 nbytes -= nblocks * SM4_BLOCK_SIZE;
215 if (walk.nbytes == walk.total && nbytes > 0) {
216 u8 keystream[SM4_BLOCK_SIZE];
218 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv);
219 crypto_xor_cpy(dst, src, keystream, nbytes);
223 err = skcipher_walk_done(&walk, nbytes);
229 static int sm4_ctr_crypt(struct skcipher_request *req)
231 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
232 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
233 struct skcipher_walk walk;
237 err = skcipher_walk_virt(&walk, req, false);
239 while ((nbytes = walk.nbytes) > 0) {
240 const u8 *src = walk.src.virt.addr;
241 u8 *dst = walk.dst.virt.addr;
242 unsigned int nblocks;
244 nblocks = nbytes / SM4_BLOCK_SIZE;
248 sm4_neon_ctr_crypt(ctx->rkey_enc, dst, src,
253 dst += nblocks * SM4_BLOCK_SIZE;
254 src += nblocks * SM4_BLOCK_SIZE;
255 nbytes -= nblocks * SM4_BLOCK_SIZE;
259 if (walk.nbytes == walk.total && nbytes > 0) {
260 u8 keystream[SM4_BLOCK_SIZE];
262 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv);
263 crypto_inc(walk.iv, SM4_BLOCK_SIZE);
264 crypto_xor_cpy(dst, src, keystream, nbytes);
268 err = skcipher_walk_done(&walk, nbytes);
274 static struct skcipher_alg sm4_algs[] = {
277 .cra_name = "ecb(sm4)",
278 .cra_driver_name = "ecb-sm4-neon",
280 .cra_blocksize = SM4_BLOCK_SIZE,
281 .cra_ctxsize = sizeof(struct sm4_ctx),
282 .cra_module = THIS_MODULE,
284 .min_keysize = SM4_KEY_SIZE,
285 .max_keysize = SM4_KEY_SIZE,
286 .setkey = sm4_setkey,
287 .encrypt = sm4_ecb_encrypt,
288 .decrypt = sm4_ecb_decrypt,
291 .cra_name = "cbc(sm4)",
292 .cra_driver_name = "cbc-sm4-neon",
294 .cra_blocksize = SM4_BLOCK_SIZE,
295 .cra_ctxsize = sizeof(struct sm4_ctx),
296 .cra_module = THIS_MODULE,
298 .min_keysize = SM4_KEY_SIZE,
299 .max_keysize = SM4_KEY_SIZE,
300 .ivsize = SM4_BLOCK_SIZE,
301 .setkey = sm4_setkey,
302 .encrypt = sm4_cbc_encrypt,
303 .decrypt = sm4_cbc_decrypt,
306 .cra_name = "cfb(sm4)",
307 .cra_driver_name = "cfb-sm4-neon",
310 .cra_ctxsize = sizeof(struct sm4_ctx),
311 .cra_module = THIS_MODULE,
313 .min_keysize = SM4_KEY_SIZE,
314 .max_keysize = SM4_KEY_SIZE,
315 .ivsize = SM4_BLOCK_SIZE,
316 .chunksize = SM4_BLOCK_SIZE,
317 .setkey = sm4_setkey,
318 .encrypt = sm4_cfb_encrypt,
319 .decrypt = sm4_cfb_decrypt,
322 .cra_name = "ctr(sm4)",
323 .cra_driver_name = "ctr-sm4-neon",
326 .cra_ctxsize = sizeof(struct sm4_ctx),
327 .cra_module = THIS_MODULE,
329 .min_keysize = SM4_KEY_SIZE,
330 .max_keysize = SM4_KEY_SIZE,
331 .ivsize = SM4_BLOCK_SIZE,
332 .chunksize = SM4_BLOCK_SIZE,
333 .setkey = sm4_setkey,
334 .encrypt = sm4_ctr_crypt,
335 .decrypt = sm4_ctr_crypt,
339 static int __init sm4_init(void)
341 return crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
344 static void __exit sm4_exit(void)
346 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
349 module_init(sm4_init);
350 module_exit(sm4_exit);
352 MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR using ARMv8 NEON");
353 MODULE_ALIAS_CRYPTO("sm4-neon");
354 MODULE_ALIAS_CRYPTO("sm4");
355 MODULE_ALIAS_CRYPTO("ecb(sm4)");
356 MODULE_ALIAS_CRYPTO("cbc(sm4)");
357 MODULE_ALIAS_CRYPTO("cfb(sm4)");
358 MODULE_ALIAS_CRYPTO("ctr(sm4)");
359 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
360 MODULE_LICENSE("GPL v2");