1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Glue code for AES implementation for SPE instructions (PPC)
5 * Based on generic implementation. The assembler module takes care
6 * about the SPE registers so it can run from interrupt context.
8 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
11 #include <crypto/aes.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/types.h>
15 #include <linux/errno.h>
16 #include <linux/crypto.h>
17 #include <asm/byteorder.h>
18 #include <asm/switch_to.h>
19 #include <crypto/algapi.h>
20 #include <crypto/xts.h>
23 * MAX_BYTES defines the number of bytes that are allowed to be processed
24 * between preempt_disable() and preempt_enable(). e500 cores can issue two
25 * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32
26 * bit unit (SU2). One of these can be a memory access that is executed via
27 * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per
28 * 16 byte block block or 25 cycles per byte. Thus 768 bytes of input data
29 * will need an estimated maximum of 20,000 cycles. Headroom for cache misses
30 * included. Even with the low end model clocked at 667 MHz this equals to a
31 * critical time window of less than 30us. The value has been chosen to
32 * process a 512 byte disk block in one or a large 1400 bytes IPsec network
39 u32 key_enc[AES_MAX_KEYLENGTH_U32];
40 u32 key_dec[AES_MAX_KEYLENGTH_U32];
45 u32 key_enc[AES_MAX_KEYLENGTH_U32];
46 u32 key_dec[AES_MAX_KEYLENGTH_U32];
47 u32 key_twk[AES_MAX_KEYLENGTH_U32];
51 extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
52 extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
53 extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
55 extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
57 extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
59 extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
61 extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
63 extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
64 u32 bytes, u8 *iv, u32 *key_twk);
65 extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
66 u32 bytes, u8 *iv, u32 *key_twk);
68 extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
69 extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
70 extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
72 extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
73 unsigned int key_len);
75 static void spe_begin(void)
77 /* disable preemption and save users SPE registers if required */
82 static void spe_end(void)
85 /* reenable preemption */
89 static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
92 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
94 if (key_len != AES_KEYSIZE_128 &&
95 key_len != AES_KEYSIZE_192 &&
96 key_len != AES_KEYSIZE_256) {
97 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
102 case AES_KEYSIZE_128:
104 ppc_expand_key_128(ctx->key_enc, in_key);
106 case AES_KEYSIZE_192:
108 ppc_expand_key_192(ctx->key_enc, in_key);
110 case AES_KEYSIZE_256:
112 ppc_expand_key_256(ctx->key_enc, in_key);
116 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
121 static int ppc_xts_setkey(struct crypto_tfm *tfm, const u8 *in_key,
122 unsigned int key_len)
124 struct ppc_xts_ctx *ctx = crypto_tfm_ctx(tfm);
127 err = xts_check_key(tfm, in_key, key_len);
133 if (key_len != AES_KEYSIZE_128 &&
134 key_len != AES_KEYSIZE_192 &&
135 key_len != AES_KEYSIZE_256) {
136 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
141 case AES_KEYSIZE_128:
143 ppc_expand_key_128(ctx->key_enc, in_key);
144 ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128);
146 case AES_KEYSIZE_192:
148 ppc_expand_key_192(ctx->key_enc, in_key);
149 ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192);
151 case AES_KEYSIZE_256:
153 ppc_expand_key_256(ctx->key_enc, in_key);
154 ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256);
158 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
163 static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
165 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
168 ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
172 static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
174 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
177 ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
181 static int ppc_ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
182 struct scatterlist *src, unsigned int nbytes)
184 struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
185 struct blkcipher_walk walk;
189 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
190 blkcipher_walk_init(&walk, dst, src, nbytes);
191 err = blkcipher_walk_virt(desc, &walk);
193 while ((nbytes = walk.nbytes)) {
194 ubytes = nbytes > MAX_BYTES ?
195 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
199 ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
200 ctx->key_enc, ctx->rounds, nbytes);
203 err = blkcipher_walk_done(desc, &walk, ubytes);
209 static int ppc_ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
210 struct scatterlist *src, unsigned int nbytes)
212 struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
213 struct blkcipher_walk walk;
217 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
218 blkcipher_walk_init(&walk, dst, src, nbytes);
219 err = blkcipher_walk_virt(desc, &walk);
221 while ((nbytes = walk.nbytes)) {
222 ubytes = nbytes > MAX_BYTES ?
223 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
227 ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
228 ctx->key_dec, ctx->rounds, nbytes);
231 err = blkcipher_walk_done(desc, &walk, ubytes);
237 static int ppc_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
238 struct scatterlist *src, unsigned int nbytes)
240 struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
241 struct blkcipher_walk walk;
245 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
246 blkcipher_walk_init(&walk, dst, src, nbytes);
247 err = blkcipher_walk_virt(desc, &walk);
249 while ((nbytes = walk.nbytes)) {
250 ubytes = nbytes > MAX_BYTES ?
251 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
255 ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
256 ctx->key_enc, ctx->rounds, nbytes, walk.iv);
259 err = blkcipher_walk_done(desc, &walk, ubytes);
265 static int ppc_cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
266 struct scatterlist *src, unsigned int nbytes)
268 struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269 struct blkcipher_walk walk;
273 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
274 blkcipher_walk_init(&walk, dst, src, nbytes);
275 err = blkcipher_walk_virt(desc, &walk);
277 while ((nbytes = walk.nbytes)) {
278 ubytes = nbytes > MAX_BYTES ?
279 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
283 ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
284 ctx->key_dec, ctx->rounds, nbytes, walk.iv);
287 err = blkcipher_walk_done(desc, &walk, ubytes);
293 static int ppc_ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
294 struct scatterlist *src, unsigned int nbytes)
296 struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
297 struct blkcipher_walk walk;
298 unsigned int pbytes, ubytes;
301 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
302 blkcipher_walk_init(&walk, dst, src, nbytes);
303 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
305 while ((pbytes = walk.nbytes)) {
306 pbytes = pbytes > MAX_BYTES ? MAX_BYTES : pbytes;
307 pbytes = pbytes == nbytes ?
308 nbytes : pbytes & ~(AES_BLOCK_SIZE - 1);
309 ubytes = walk.nbytes - pbytes;
312 ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr,
313 ctx->key_enc, ctx->rounds, pbytes , walk.iv);
317 err = blkcipher_walk_done(desc, &walk, ubytes);
323 static int ppc_xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
324 struct scatterlist *src, unsigned int nbytes)
326 struct ppc_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
327 struct blkcipher_walk walk;
332 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
333 blkcipher_walk_init(&walk, dst, src, nbytes);
334 err = blkcipher_walk_virt(desc, &walk);
337 while ((nbytes = walk.nbytes)) {
338 ubytes = nbytes > MAX_BYTES ?
339 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
343 ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
344 ctx->key_enc, ctx->rounds, nbytes, walk.iv, twk);
348 err = blkcipher_walk_done(desc, &walk, ubytes);
354 static int ppc_xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
355 struct scatterlist *src, unsigned int nbytes)
357 struct ppc_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
358 struct blkcipher_walk walk;
363 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
364 blkcipher_walk_init(&walk, dst, src, nbytes);
365 err = blkcipher_walk_virt(desc, &walk);
368 while ((nbytes = walk.nbytes)) {
369 ubytes = nbytes > MAX_BYTES ?
370 nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1);
374 ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
375 ctx->key_dec, ctx->rounds, nbytes, walk.iv, twk);
379 err = blkcipher_walk_done(desc, &walk, ubytes);
386 * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen
387 * because the e500 platform can handle unaligned reads/writes very efficently.
388 * This improves IPsec thoughput by another few percent. Additionally we assume
389 * that AES context is always aligned to at least 8 bytes because it is created
390 * with kmalloc() in the crypto infrastructure
393 static struct crypto_alg aes_algs[] = { {
395 .cra_driver_name = "aes-ppc-spe",
397 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
398 .cra_blocksize = AES_BLOCK_SIZE,
399 .cra_ctxsize = sizeof(struct ppc_aes_ctx),
401 .cra_module = THIS_MODULE,
404 .cia_min_keysize = AES_MIN_KEY_SIZE,
405 .cia_max_keysize = AES_MAX_KEY_SIZE,
406 .cia_setkey = ppc_aes_setkey,
407 .cia_encrypt = ppc_aes_encrypt,
408 .cia_decrypt = ppc_aes_decrypt
412 .cra_name = "ecb(aes)",
413 .cra_driver_name = "ecb-ppc-spe",
415 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
416 .cra_blocksize = AES_BLOCK_SIZE,
417 .cra_ctxsize = sizeof(struct ppc_aes_ctx),
419 .cra_type = &crypto_blkcipher_type,
420 .cra_module = THIS_MODULE,
423 .min_keysize = AES_MIN_KEY_SIZE,
424 .max_keysize = AES_MAX_KEY_SIZE,
425 .ivsize = AES_BLOCK_SIZE,
426 .setkey = ppc_aes_setkey,
427 .encrypt = ppc_ecb_encrypt,
428 .decrypt = ppc_ecb_decrypt,
432 .cra_name = "cbc(aes)",
433 .cra_driver_name = "cbc-ppc-spe",
435 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
436 .cra_blocksize = AES_BLOCK_SIZE,
437 .cra_ctxsize = sizeof(struct ppc_aes_ctx),
439 .cra_type = &crypto_blkcipher_type,
440 .cra_module = THIS_MODULE,
443 .min_keysize = AES_MIN_KEY_SIZE,
444 .max_keysize = AES_MAX_KEY_SIZE,
445 .ivsize = AES_BLOCK_SIZE,
446 .setkey = ppc_aes_setkey,
447 .encrypt = ppc_cbc_encrypt,
448 .decrypt = ppc_cbc_decrypt,
452 .cra_name = "ctr(aes)",
453 .cra_driver_name = "ctr-ppc-spe",
455 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
457 .cra_ctxsize = sizeof(struct ppc_aes_ctx),
459 .cra_type = &crypto_blkcipher_type,
460 .cra_module = THIS_MODULE,
463 .min_keysize = AES_MIN_KEY_SIZE,
464 .max_keysize = AES_MAX_KEY_SIZE,
465 .ivsize = AES_BLOCK_SIZE,
466 .setkey = ppc_aes_setkey,
467 .encrypt = ppc_ctr_crypt,
468 .decrypt = ppc_ctr_crypt,
472 .cra_name = "xts(aes)",
473 .cra_driver_name = "xts-ppc-spe",
475 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
476 .cra_blocksize = AES_BLOCK_SIZE,
477 .cra_ctxsize = sizeof(struct ppc_xts_ctx),
479 .cra_type = &crypto_blkcipher_type,
480 .cra_module = THIS_MODULE,
483 .min_keysize = AES_MIN_KEY_SIZE * 2,
484 .max_keysize = AES_MAX_KEY_SIZE * 2,
485 .ivsize = AES_BLOCK_SIZE,
486 .setkey = ppc_xts_setkey,
487 .encrypt = ppc_xts_encrypt,
488 .decrypt = ppc_xts_decrypt,
493 static int __init ppc_aes_mod_init(void)
495 return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
498 static void __exit ppc_aes_mod_fini(void)
500 crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
503 module_init(ppc_aes_mod_init);
504 module_exit(ppc_aes_mod_fini);
506 MODULE_LICENSE("GPL");
507 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized");
509 MODULE_ALIAS_CRYPTO("aes");
510 MODULE_ALIAS_CRYPTO("ecb(aes)");
511 MODULE_ALIAS_CRYPTO("cbc(aes)");
512 MODULE_ALIAS_CRYPTO("ctr(aes)");
513 MODULE_ALIAS_CRYPTO("xts(aes)");
514 MODULE_ALIAS_CRYPTO("aes-ppc-spe");