arch/x86/crypto/twofish_avx_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for AVX assembler version of Twofish Cipher
   4  *
   5  * Copyright (C) 2012 Johannes Goetzfried
   6  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   7  *
   8  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/types.h>
  13 #include <linux/crypto.h>
  14 #include <linux/err.h>
  15 #include <crypto/algapi.h>
  16 #include <crypto/internal/simd.h>
  17 #include <crypto/twofish.h>
  18 #include <crypto/xts.h>
  19 #include <asm/crypto/glue_helper.h>
  20 #include <asm/crypto/twofish.h>
  21
  22 #define TWOFISH_PARALLEL_BLOCKS 8
  23
  24 /* 8-way parallel cipher functions */
  25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src);
  26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src);
  27
  28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src);
  29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src,
  30                                  le128 *iv);
  31
  32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src,
  33                                      le128 *iv);
  34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src,
  35                                      le128 *iv);
  36
  37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
  38                                    const u8 *key, unsigned int keylen)
  39 {
  40         return twofish_setkey(&tfm->base, key, keylen);
  41 }
  42
  43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
  44 {
  45         __twofish_enc_blk_3way(ctx, dst, src, false);
  46 }
  47
  48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  49 {
  50         glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk);
  51 }
  52
  53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  54 {
  55         glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk);
  56 }
  57
  58 struct twofish_xts_ctx {
  59         struct twofish_ctx tweak_ctx;
  60         struct twofish_ctx crypt_ctx;
  61 };
  62
  63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
  64                               unsigned int keylen)
  65 {
  66         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  67         u32 *flags = &tfm->base.crt_flags;
  68         int err;
  69
  70         err = xts_verify_key(tfm, key, keylen);
  71         if (err)
  72                 return err;
  73
  74         /* first half of xts-key is for crypt */
  75         err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
  76         if (err)
  77                 return err;
  78
  79         /* second half of xts-key is for tweak */
  80         return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
  81                                 flags);
  82 }
  83
  84 static const struct common_glue_ctx twofish_enc = {
  85         .num_funcs = 3,
  86         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
  87
  88         .funcs = { {
  89                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
  90                 .fn_u = { .ecb = twofish_ecb_enc_8way }
  91         }, {
  92                 .num_blocks = 3,
  93                 .fn_u = { .ecb = twofish_enc_blk_3way }
  94         }, {
  95                 .num_blocks = 1,
  96                 .fn_u = { .ecb = twofish_enc_blk }
  97         } }
  98 };
  99
 100 static const struct common_glue_ctx twofish_ctr = {
 101         .num_funcs = 3,
 102         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 103
 104         .funcs = { {
 105                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 106                 .fn_u = { .ctr = twofish_ctr_8way }
 107         }, {
 108                 .num_blocks = 3,
 109                 .fn_u = { .ctr = twofish_enc_blk_ctr_3way }
 110         }, {
 111                 .num_blocks = 1,
 112                 .fn_u = { .ctr = twofish_enc_blk_ctr }
 113         } }
 114 };
 115
 116 static const struct common_glue_ctx twofish_enc_xts = {
 117         .num_funcs = 2,
 118         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 119
 120         .funcs = { {
 121                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 122                 .fn_u = { .xts = twofish_xts_enc_8way }
 123         }, {
 124                 .num_blocks = 1,
 125                 .fn_u = { .xts = twofish_xts_enc }
 126         } }
 127 };
 128
 129 static const struct common_glue_ctx twofish_dec = {
 130         .num_funcs = 3,
 131         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 132
 133         .funcs = { {
 134                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 135                 .fn_u = { .ecb = twofish_ecb_dec_8way }
 136         }, {
 137                 .num_blocks = 3,
 138                 .fn_u = { .ecb = twofish_dec_blk_3way }
 139         }, {
 140                 .num_blocks = 1,
 141                 .fn_u = { .ecb = twofish_dec_blk }
 142         } }
 143 };
 144
 145 static const struct common_glue_ctx twofish_dec_cbc = {
 146         .num_funcs = 3,
 147         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 148
 149         .funcs = { {
 150                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 151                 .fn_u = { .cbc = twofish_cbc_dec_8way }
 152         }, {
 153                 .num_blocks = 3,
 154                 .fn_u = { .cbc = twofish_dec_blk_cbc_3way }
 155         }, {
 156                 .num_blocks = 1,
 157                 .fn_u = { .cbc = twofish_dec_blk }
 158         } }
 159 };
 160
 161 static const struct common_glue_ctx twofish_dec_xts = {
 162         .num_funcs = 2,
 163         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 164
 165         .funcs = { {
 166                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 167                 .fn_u = { .xts = twofish_xts_dec_8way }
 168         }, {
 169                 .num_blocks = 1,
 170                 .fn_u = { .xts = twofish_xts_dec }
 171         } }
 172 };
 173
 174 static int ecb_encrypt(struct skcipher_request *req)
 175 {
 176         return glue_ecb_req_128bit(&twofish_enc, req);
 177 }
 178
 179 static int ecb_decrypt(struct skcipher_request *req)
 180 {
 181         return glue_ecb_req_128bit(&twofish_dec, req);
 182 }
 183
 184 static int cbc_encrypt(struct skcipher_request *req)
 185 {
 186         return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req);
 187 }
 188
 189 static int cbc_decrypt(struct skcipher_request *req)
 190 {
 191         return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
 192 }
 193
 194 static int ctr_crypt(struct skcipher_request *req)
 195 {
 196         return glue_ctr_req_128bit(&twofish_ctr, req);
 197 }
 198
 199 static int xts_encrypt(struct skcipher_request *req)
 200 {
 201         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 202         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 203
 204         return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk,
 205                                    &ctx->tweak_ctx, &ctx->crypt_ctx, false);
 206 }
 207
 208 static int xts_decrypt(struct skcipher_request *req)
 209 {
 210         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 211         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 212
 213         return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk,
 214                                    &ctx->tweak_ctx, &ctx->crypt_ctx, true);
 215 }
 216
 217 static struct skcipher_alg twofish_algs[] = {
 218         {
 219                 .base.cra_name          = "__ecb(twofish)",
 220                 .base.cra_driver_name   = "__ecb-twofish-avx",
 221                 .base.cra_priority      = 400,
 222                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 223                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 224                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 225                 .base.cra_module        = THIS_MODULE,
 226                 .min_keysize            = TF_MIN_KEY_SIZE,
 227                 .max_keysize            = TF_MAX_KEY_SIZE,
 228                 .setkey                 = twofish_setkey_skcipher,
 229                 .encrypt                = ecb_encrypt,
 230                 .decrypt                = ecb_decrypt,
 231         }, {
 232                 .base.cra_name          = "__cbc(twofish)",
 233                 .base.cra_driver_name   = "__cbc-twofish-avx",
 234                 .base.cra_priority      = 400,
 235                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 236                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 237                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 238                 .base.cra_module        = THIS_MODULE,
 239                 .min_keysize            = TF_MIN_KEY_SIZE,
 240                 .max_keysize            = TF_MAX_KEY_SIZE,
 241                 .ivsize                 = TF_BLOCK_SIZE,
 242                 .setkey                 = twofish_setkey_skcipher,
 243                 .encrypt                = cbc_encrypt,
 244                 .decrypt                = cbc_decrypt,
 245         }, {
 246                 .base.cra_name          = "__ctr(twofish)",
 247                 .base.cra_driver_name   = "__ctr-twofish-avx",
 248                 .base.cra_priority      = 400,
 249                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 250                 .base.cra_blocksize     = 1,
 251                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 252                 .base.cra_module        = THIS_MODULE,
 253                 .min_keysize            = TF_MIN_KEY_SIZE,
 254                 .max_keysize            = TF_MAX_KEY_SIZE,
 255                 .ivsize                 = TF_BLOCK_SIZE,
 256                 .chunksize              = TF_BLOCK_SIZE,
 257                 .setkey                 = twofish_setkey_skcipher,
 258                 .encrypt                = ctr_crypt,
 259                 .decrypt                = ctr_crypt,
 260         }, {
 261                 .base.cra_name          = "__xts(twofish)",
 262                 .base.cra_driver_name   = "__xts-twofish-avx",
 263                 .base.cra_priority      = 400,
 264                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 265                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 266                 .base.cra_ctxsize       = sizeof(struct twofish_xts_ctx),
 267                 .base.cra_module        = THIS_MODULE,
 268                 .min_keysize            = 2 * TF_MIN_KEY_SIZE,
 269                 .max_keysize            = 2 * TF_MAX_KEY_SIZE,
 270                 .ivsize                 = TF_BLOCK_SIZE,
 271                 .setkey                 = xts_twofish_setkey,
 272                 .encrypt                = xts_encrypt,
 273                 .decrypt                = xts_decrypt,
 274         },
 275 };
 276
 277 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
 278
 279 static int __init twofish_init(void)
 280 {
 281         const char *feature_name;
 282
 283         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
 284                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 285                 return -ENODEV;
 286         }
 287
 288         return simd_register_skciphers_compat(twofish_algs,
 289                                               ARRAY_SIZE(twofish_algs),
 290                                               twofish_simd_algs);
 291 }
 292
 293 static void __exit twofish_exit(void)
 294 {
 295         simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
 296                                   twofish_simd_algs);
 297 }
 298
 299 module_init(twofish_init);
 300 module_exit(twofish_exit);
 301
 302 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
 303 MODULE_LICENSE("GPL");
 304 MODULE_ALIAS_CRYPTO("twofish");