arch/x86/crypto/camellia_aesni_avx2_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
   4  *
   5  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6  */
   7
   8 #include <asm/crypto/camellia.h>
   9 #include <asm/crypto/glue_helper.h>
  10 #include <crypto/algapi.h>
  11 #include <crypto/internal/simd.h>
  12 #include <crypto/xts.h>
  13 #include <linux/crypto.h>
  14 #include <linux/err.h>
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17
  18 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  19 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
  20
  21 /* 32-way AVX2/AES-NI parallel cipher functions */
  22 asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src);
  23 asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src);
  24
  25 asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src);
  26 asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src,
  27                                    le128 *iv);
  28
  29 asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src,
  30                                        le128 *iv);
  31 asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src,
  32                                        le128 *iv);
  33
  34 static const struct common_glue_ctx camellia_enc = {
  35         .num_funcs = 4,
  36         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  37
  38         .funcs = { {
  39                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  40                 .fn_u = { .ecb = camellia_ecb_enc_32way }
  41         }, {
  42                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  43                 .fn_u = { .ecb = camellia_ecb_enc_16way }
  44         }, {
  45                 .num_blocks = 2,
  46                 .fn_u = { .ecb = camellia_enc_blk_2way }
  47         }, {
  48                 .num_blocks = 1,
  49                 .fn_u = { .ecb = camellia_enc_blk }
  50         } }
  51 };
  52
  53 static const struct common_glue_ctx camellia_ctr = {
  54         .num_funcs = 4,
  55         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  56
  57         .funcs = { {
  58                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  59                 .fn_u = { .ctr = camellia_ctr_32way }
  60         }, {
  61                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  62                 .fn_u = { .ctr = camellia_ctr_16way }
  63         }, {
  64                 .num_blocks = 2,
  65                 .fn_u = { .ctr = camellia_crypt_ctr_2way }
  66         }, {
  67                 .num_blocks = 1,
  68                 .fn_u = { .ctr = camellia_crypt_ctr }
  69         } }
  70 };
  71
  72 static const struct common_glue_ctx camellia_enc_xts = {
  73         .num_funcs = 3,
  74         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  75
  76         .funcs = { {
  77                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  78                 .fn_u = { .xts = camellia_xts_enc_32way }
  79         }, {
  80                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  81                 .fn_u = { .xts = camellia_xts_enc_16way }
  82         }, {
  83                 .num_blocks = 1,
  84                 .fn_u = { .xts = camellia_xts_enc }
  85         } }
  86 };
  87
  88 static const struct common_glue_ctx camellia_dec = {
  89         .num_funcs = 4,
  90         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  91
  92         .funcs = { {
  93                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  94                 .fn_u = { .ecb = camellia_ecb_dec_32way }
  95         }, {
  96                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  97                 .fn_u = { .ecb = camellia_ecb_dec_16way }
  98         }, {
  99                 .num_blocks = 2,
 100                 .fn_u = { .ecb = camellia_dec_blk_2way }
 101         }, {
 102                 .num_blocks = 1,
 103                 .fn_u = { .ecb = camellia_dec_blk }
 104         } }
 105 };
 106
 107 static const struct common_glue_ctx camellia_dec_cbc = {
 108         .num_funcs = 4,
 109         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 110
 111         .funcs = { {
 112                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 113                 .fn_u = { .cbc = camellia_cbc_dec_32way }
 114         }, {
 115                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 116                 .fn_u = { .cbc = camellia_cbc_dec_16way }
 117         }, {
 118                 .num_blocks = 2,
 119                 .fn_u = { .cbc = camellia_decrypt_cbc_2way }
 120         }, {
 121                 .num_blocks = 1,
 122                 .fn_u = { .cbc = camellia_dec_blk }
 123         } }
 124 };
 125
 126 static const struct common_glue_ctx camellia_dec_xts = {
 127         .num_funcs = 3,
 128         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 129
 130         .funcs = { {
 131                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 132                 .fn_u = { .xts = camellia_xts_dec_32way }
 133         }, {
 134                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 135                 .fn_u = { .xts = camellia_xts_dec_16way }
 136         }, {
 137                 .num_blocks = 1,
 138                 .fn_u = { .xts = camellia_xts_dec }
 139         } }
 140 };
 141
 142 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
 143                            unsigned int keylen)
 144 {
 145         return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen,
 146                                  &tfm->base.crt_flags);
 147 }
 148
 149 static int ecb_encrypt(struct skcipher_request *req)
 150 {
 151         return glue_ecb_req_128bit(&camellia_enc, req);
 152 }
 153
 154 static int ecb_decrypt(struct skcipher_request *req)
 155 {
 156         return glue_ecb_req_128bit(&camellia_dec, req);
 157 }
 158
 159 static int cbc_encrypt(struct skcipher_request *req)
 160 {
 161         return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
 162 }
 163
 164 static int cbc_decrypt(struct skcipher_request *req)
 165 {
 166         return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
 167 }
 168
 169 static int ctr_crypt(struct skcipher_request *req)
 170 {
 171         return glue_ctr_req_128bit(&camellia_ctr, req);
 172 }
 173
 174 static int xts_encrypt(struct skcipher_request *req)
 175 {
 176         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 177         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 178
 179         return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
 180                                    &ctx->tweak_ctx, &ctx->crypt_ctx, false);
 181 }
 182
 183 static int xts_decrypt(struct skcipher_request *req)
 184 {
 185         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 186         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 187
 188         return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
 189                                    &ctx->tweak_ctx, &ctx->crypt_ctx, true);
 190 }
 191
 192 static struct skcipher_alg camellia_algs[] = {
 193         {
 194                 .base.cra_name          = "__ecb(camellia)",
 195                 .base.cra_driver_name   = "__ecb-camellia-aesni-avx2",
 196                 .base.cra_priority      = 500,
 197                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 198                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 199                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 200                 .base.cra_module        = THIS_MODULE,
 201                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 202                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 203                 .setkey                 = camellia_setkey,
 204                 .encrypt                = ecb_encrypt,
 205                 .decrypt                = ecb_decrypt,
 206         }, {
 207                 .base.cra_name          = "__cbc(camellia)",
 208                 .base.cra_driver_name   = "__cbc-camellia-aesni-avx2",
 209                 .base.cra_priority      = 500,
 210                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 211                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 212                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 213                 .base.cra_module        = THIS_MODULE,
 214                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 215                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 216                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 217                 .setkey                 = camellia_setkey,
 218                 .encrypt                = cbc_encrypt,
 219                 .decrypt                = cbc_decrypt,
 220         }, {
 221                 .base.cra_name          = "__ctr(camellia)",
 222                 .base.cra_driver_name   = "__ctr-camellia-aesni-avx2",
 223                 .base.cra_priority      = 500,
 224                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 225                 .base.cra_blocksize     = 1,
 226                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 227                 .base.cra_module        = THIS_MODULE,
 228                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 229                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 230                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 231                 .chunksize              = CAMELLIA_BLOCK_SIZE,
 232                 .setkey                 = camellia_setkey,
 233                 .encrypt                = ctr_crypt,
 234                 .decrypt                = ctr_crypt,
 235         }, {
 236                 .base.cra_name          = "__xts(camellia)",
 237                 .base.cra_driver_name   = "__xts-camellia-aesni-avx2",
 238                 .base.cra_priority      = 500,
 239                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 240                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 241                 .base.cra_ctxsize       = sizeof(struct camellia_xts_ctx),
 242                 .base.cra_module        = THIS_MODULE,
 243                 .min_keysize            = 2 * CAMELLIA_MIN_KEY_SIZE,
 244                 .max_keysize            = 2 * CAMELLIA_MAX_KEY_SIZE,
 245                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 246                 .setkey                 = xts_camellia_setkey,
 247                 .encrypt                = xts_encrypt,
 248                 .decrypt                = xts_decrypt,
 249         },
 250 };
 251
 252 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
 253
 254 static int __init camellia_aesni_init(void)
 255 {
 256         const char *feature_name;
 257
 258         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 259             !boot_cpu_has(X86_FEATURE_AVX2) ||
 260             !boot_cpu_has(X86_FEATURE_AES) ||
 261             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 262                 pr_info("AVX2 or AES-NI instructions are not detected.\n");
 263                 return -ENODEV;
 264         }
 265
 266         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 267                                 &feature_name)) {
 268                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 269                 return -ENODEV;
 270         }
 271
 272         return simd_register_skciphers_compat(camellia_algs,
 273                                               ARRAY_SIZE(camellia_algs),
 274                                               camellia_simd_algs);
 275 }
 276
 277 static void __exit camellia_aesni_fini(void)
 278 {
 279         simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
 280                                   camellia_simd_algs);
 281 }
 282
 283 module_init(camellia_aesni_init);
 284 module_exit(camellia_aesni_fini);
 285
 286 MODULE_LICENSE("GPL");
 287 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
 288 MODULE_ALIAS_CRYPTO("camellia");
 289 MODULE_ALIAS_CRYPTO("camellia-asm");