arch/x86/crypto/twofish_avx_glue.c

   1 /*
   2  * Glue Code for AVX assembler version of Twofish Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   6  *
   7  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  22  * USA
  23  *
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <linux/crypto.h>
  29 #include <linux/err.h>
  30 #include <crypto/algapi.h>
  31 #include <crypto/internal/simd.h>
  32 #include <crypto/twofish.h>
  33 #include <crypto/xts.h>
  34 #include <asm/crypto/glue_helper.h>
  35 #include <asm/crypto/twofish.h>
  36
  37 #define TWOFISH_PARALLEL_BLOCKS 8
  38
  39 /* 8-way parallel cipher functions */
  40 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
  41                                      const u8 *src);
  42 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  43                                      const u8 *src);
  44
  45 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  46                                      const u8 *src);
  47 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
  48                                  const u8 *src, le128 *iv);
  49
  50 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
  51                                      const u8 *src, le128 *iv);
  52 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  53                                      const u8 *src, le128 *iv);
  54
  55 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
  56                                    const u8 *key, unsigned int keylen)
  57 {
  58         return twofish_setkey(&tfm->base, key, keylen);
  59 }
  60
  61 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  62                                         const u8 *src)
  63 {
  64         __twofish_enc_blk_3way(ctx, dst, src, false);
  65 }
  66
  67 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  68 {
  69         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  70                                   GLUE_FUNC_CAST(twofish_enc_blk));
  71 }
  72
  73 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  74 {
  75         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  76                                   GLUE_FUNC_CAST(twofish_dec_blk));
  77 }
  78
  79 struct twofish_xts_ctx {
  80         struct twofish_ctx tweak_ctx;
  81         struct twofish_ctx crypt_ctx;
  82 };
  83
  84 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
  85                               unsigned int keylen)
  86 {
  87         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  88         u32 *flags = &tfm->base.crt_flags;
  89         int err;
  90
  91         err = xts_verify_key(tfm, key, keylen);
  92         if (err)
  93                 return err;
  94
  95         /* first half of xts-key is for crypt */
  96         err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
  97         if (err)
  98                 return err;
  99
 100         /* second half of xts-key is for tweak */
 101         return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
 102                                 flags);
 103 }
 104
 105 static const struct common_glue_ctx twofish_enc = {
 106         .num_funcs = 3,
 107         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 108
 109         .funcs = { {
 110                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 111                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
 112         }, {
 113                 .num_blocks = 3,
 114                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
 115         }, {
 116                 .num_blocks = 1,
 117                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
 118         } }
 119 };
 120
 121 static const struct common_glue_ctx twofish_ctr = {
 122         .num_funcs = 3,
 123         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 124
 125         .funcs = { {
 126                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 127                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
 128         }, {
 129                 .num_blocks = 3,
 130                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
 131         }, {
 132                 .num_blocks = 1,
 133                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
 134         } }
 135 };
 136
 137 static const struct common_glue_ctx twofish_enc_xts = {
 138         .num_funcs = 2,
 139         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 140
 141         .funcs = { {
 142                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 143                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
 144         }, {
 145                 .num_blocks = 1,
 146                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
 147         } }
 148 };
 149
 150 static const struct common_glue_ctx twofish_dec = {
 151         .num_funcs = 3,
 152         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 153
 154         .funcs = { {
 155                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 156                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
 157         }, {
 158                 .num_blocks = 3,
 159                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
 160         }, {
 161                 .num_blocks = 1,
 162                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
 163         } }
 164 };
 165
 166 static const struct common_glue_ctx twofish_dec_cbc = {
 167         .num_funcs = 3,
 168         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 169
 170         .funcs = { {
 171                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 172                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
 173         }, {
 174                 .num_blocks = 3,
 175                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
 176         }, {
 177                 .num_blocks = 1,
 178                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
 179         } }
 180 };
 181
 182 static const struct common_glue_ctx twofish_dec_xts = {
 183         .num_funcs = 2,
 184         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 185
 186         .funcs = { {
 187                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 188                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
 189         }, {
 190                 .num_blocks = 1,
 191                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
 192         } }
 193 };
 194
 195 static int ecb_encrypt(struct skcipher_request *req)
 196 {
 197         return glue_ecb_req_128bit(&twofish_enc, req);
 198 }
 199
 200 static int ecb_decrypt(struct skcipher_request *req)
 201 {
 202         return glue_ecb_req_128bit(&twofish_dec, req);
 203 }
 204
 205 static int cbc_encrypt(struct skcipher_request *req)
 206 {
 207         return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
 208                                            req);
 209 }
 210
 211 static int cbc_decrypt(struct skcipher_request *req)
 212 {
 213         return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
 214 }
 215
 216 static int ctr_crypt(struct skcipher_request *req)
 217 {
 218         return glue_ctr_req_128bit(&twofish_ctr, req);
 219 }
 220
 221 static int xts_encrypt(struct skcipher_request *req)
 222 {
 223         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 224         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 225
 226         return glue_xts_req_128bit(&twofish_enc_xts, req,
 227                                    XTS_TWEAK_CAST(twofish_enc_blk),
 228                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 229 }
 230
 231 static int xts_decrypt(struct skcipher_request *req)
 232 {
 233         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 234         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 235
 236         return glue_xts_req_128bit(&twofish_dec_xts, req,
 237                                    XTS_TWEAK_CAST(twofish_enc_blk),
 238                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 239 }
 240
 241 static struct skcipher_alg twofish_algs[] = {
 242         {
 243                 .base.cra_name          = "__ecb(twofish)",
 244                 .base.cra_driver_name   = "__ecb-twofish-avx",
 245                 .base.cra_priority      = 400,
 246                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 247                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 248                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 249                 .base.cra_module        = THIS_MODULE,
 250                 .min_keysize            = TF_MIN_KEY_SIZE,
 251                 .max_keysize            = TF_MAX_KEY_SIZE,
 252                 .setkey                 = twofish_setkey_skcipher,
 253                 .encrypt                = ecb_encrypt,
 254                 .decrypt                = ecb_decrypt,
 255         }, {
 256                 .base.cra_name          = "__cbc(twofish)",
 257                 .base.cra_driver_name   = "__cbc-twofish-avx",
 258                 .base.cra_priority      = 400,
 259                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 260                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 261                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 262                 .base.cra_module        = THIS_MODULE,
 263                 .min_keysize            = TF_MIN_KEY_SIZE,
 264                 .max_keysize            = TF_MAX_KEY_SIZE,
 265                 .ivsize                 = TF_BLOCK_SIZE,
 266                 .setkey                 = twofish_setkey_skcipher,
 267                 .encrypt                = cbc_encrypt,
 268                 .decrypt                = cbc_decrypt,
 269         }, {
 270                 .base.cra_name          = "__ctr(twofish)",
 271                 .base.cra_driver_name   = "__ctr-twofish-avx",
 272                 .base.cra_priority      = 400,
 273                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 274                 .base.cra_blocksize     = 1,
 275                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 276                 .base.cra_module        = THIS_MODULE,
 277                 .min_keysize            = TF_MIN_KEY_SIZE,
 278                 .max_keysize            = TF_MAX_KEY_SIZE,
 279                 .ivsize                 = TF_BLOCK_SIZE,
 280                 .chunksize              = TF_BLOCK_SIZE,
 281                 .setkey                 = twofish_setkey_skcipher,
 282                 .encrypt                = ctr_crypt,
 283                 .decrypt                = ctr_crypt,
 284         }, {
 285                 .base.cra_name          = "__xts(twofish)",
 286                 .base.cra_driver_name   = "__xts-twofish-avx",
 287                 .base.cra_priority      = 400,
 288                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 289                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 290                 .base.cra_ctxsize       = sizeof(struct twofish_xts_ctx),
 291                 .base.cra_module        = THIS_MODULE,
 292                 .min_keysize            = 2 * TF_MIN_KEY_SIZE,
 293                 .max_keysize            = 2 * TF_MAX_KEY_SIZE,
 294                 .ivsize                 = TF_BLOCK_SIZE,
 295                 .setkey                 = xts_twofish_setkey,
 296                 .encrypt                = xts_encrypt,
 297                 .decrypt                = xts_decrypt,
 298         },
 299 };
 300
 301 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
 302
 303 static int __init twofish_init(void)
 304 {
 305         const char *feature_name;
 306
 307         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
 308                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 309                 return -ENODEV;
 310         }
 311
 312         return simd_register_skciphers_compat(twofish_algs,
 313                                               ARRAY_SIZE(twofish_algs),
 314                                               twofish_simd_algs);
 315 }
 316
 317 static void __exit twofish_exit(void)
 318 {
 319         simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
 320                                   twofish_simd_algs);
 321 }
 322
 323 module_init(twofish_init);
 324 module_exit(twofish_exit);
 325
 326 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
 327 MODULE_LICENSE("GPL");
 328 MODULE_ALIAS_CRYPTO("twofish");