arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/module.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <crypto/b128ops.h>
  31 #include <crypto/gcm.h>
  32 #include <crypto/xts.h>
  33 #include <asm/cpu_device_id.h>
  34 #include <asm/fpu/api.h>
  35 #include <asm/crypto/aes.h>
  36 #include <crypto/scatterwalk.h>
  37 #include <crypto/internal/aead.h>
  38 #include <crypto/internal/simd.h>
  39 #include <crypto/internal/skcipher.h>
  40 #include <linux/workqueue.h>
  41 #include <linux/spinlock.h>
  42 #ifdef CONFIG_X86_64
  43 #include <asm/crypto/glue_helper.h>
  44 #endif
  45
  46
  47 #define AESNI_ALIGN     16
  48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  49 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  50 #define RFC4106_HASH_SUBKEY_SIZE 16
  51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  54
  55 /* This data is stored at the end of the crypto_tfm struct.
  56  * It's a type of per "session" data storage location.
  57  * This needs to be 16 byte aligned.
  58  */
  59 struct aesni_rfc4106_gcm_ctx {
  60         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  61         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  62         u8 nonce[4];
  63 };
  64
  65 struct generic_gcmaes_ctx {
  66         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  67         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  68 };
  69
  70 struct aesni_xts_ctx {
  71         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  72         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  73 };
  74
  75 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  76                              unsigned int key_len);
  77 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  78                           const u8 *in);
  79 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  80                           const u8 *in);
  81 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  82                               const u8 *in, unsigned int len);
  83 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  84                               const u8 *in, unsigned int len);
  85 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  86                               const u8 *in, unsigned int len, u8 *iv);
  87 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  88                               const u8 *in, unsigned int len, u8 *iv);
  89
  90 int crypto_fpu_init(void);
  91 void crypto_fpu_exit(void);
  92
  93 #define AVX_GEN2_OPTSIZE 640
  94 #define AVX_GEN4_OPTSIZE 4096
  95
  96 #ifdef CONFIG_X86_64
  97
  98 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
  99                               const u8 *in, unsigned int len, u8 *iv);
 100 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 101                               const u8 *in, unsigned int len, u8 *iv);
 102
 103 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
 104                                  const u8 *in, bool enc, u8 *iv);
 105
 106 /* asmlinkage void aesni_gcm_enc()
 107  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 108  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 109  * const u8 *in, Plaintext input
 110  * unsigned long plaintext_len, Length of data in bytes for encryption.
 111  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 112  *         16-byte aligned pointer.
 113  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 114  * const u8 *aad, Additional Authentication Data (AAD)
 115  * unsigned long aad_len, Length of AAD in bytes.
 116  * u8 *auth_tag, Authenticated Tag output.
 117  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 118  *          Valid values are 16 (most likely), 12 or 8.
 119  */
 120 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 121                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 122                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 123                         u8 *auth_tag, unsigned long auth_tag_len);
 124
 125 /* asmlinkage void aesni_gcm_dec()
 126  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 127  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 128  * const u8 *in, Ciphertext input
 129  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 130  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 131  *         16-byte aligned pointer.
 132  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 133  * const u8 *aad, Additional Authentication Data (AAD)
 134  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 135  * to be 8 or 12 bytes
 136  * u8 *auth_tag, Authenticated Tag output.
 137  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 138  * Valid values are 16 (most likely), 12 or 8.
 139  */
 140 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 141                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 142                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 143                         u8 *auth_tag, unsigned long auth_tag_len);
 144
 145
 146 #ifdef CONFIG_AS_AVX
 147 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 148                 void *keys, u8 *out, unsigned int num_bytes);
 149 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 150                 void *keys, u8 *out, unsigned int num_bytes);
 151 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 152                 void *keys, u8 *out, unsigned int num_bytes);
 153 /*
 154  * asmlinkage void aesni_gcm_precomp_avx_gen2()
 155  * gcm_data *my_ctx_data, context data
 156  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 157  */
 158 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
 159
 160 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
 161                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 162                         const u8 *aad, unsigned long aad_len,
 163                         u8 *auth_tag, unsigned long auth_tag_len);
 164
 165 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
 166                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 167                         const u8 *aad, unsigned long aad_len,
 168                         u8 *auth_tag, unsigned long auth_tag_len);
 169
 170 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
 171                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 172                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 173                         u8 *auth_tag, unsigned long auth_tag_len)
 174 {
 175         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 176         if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
 177                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 178                                 aad_len, auth_tag, auth_tag_len);
 179         } else {
 180                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 181                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 182                                         aad_len, auth_tag, auth_tag_len);
 183         }
 184 }
 185
 186 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
 187                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 188                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 189                         u8 *auth_tag, unsigned long auth_tag_len)
 190 {
 191         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 192         if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 193                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
 194                                 aad_len, auth_tag, auth_tag_len);
 195         } else {
 196                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 197                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 198                                         aad_len, auth_tag, auth_tag_len);
 199         }
 200 }
 201 #endif
 202
 203 #ifdef CONFIG_AS_AVX2
 204 /*
 205  * asmlinkage void aesni_gcm_precomp_avx_gen4()
 206  * gcm_data *my_ctx_data, context data
 207  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 208  */
 209 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
 210
 211 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
 212                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 213                         const u8 *aad, unsigned long aad_len,
 214                         u8 *auth_tag, unsigned long auth_tag_len);
 215
 216 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
 217                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 218                         const u8 *aad, unsigned long aad_len,
 219                         u8 *auth_tag, unsigned long auth_tag_len);
 220
 221 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
 222                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 223                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 224                         u8 *auth_tag, unsigned long auth_tag_len)
 225 {
 226        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 227         if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 228                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 229                                 aad_len, auth_tag, auth_tag_len);
 230         } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
 231                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 232                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 233                                         aad_len, auth_tag, auth_tag_len);
 234         } else {
 235                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 236                 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
 237                                         aad_len, auth_tag, auth_tag_len);
 238         }
 239 }
 240
 241 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
 242                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 243                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 244                         u8 *auth_tag, unsigned long auth_tag_len)
 245 {
 246        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 247         if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 248                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
 249                                 aad, aad_len, auth_tag, auth_tag_len);
 250         } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
 251                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 252                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 253                                         aad_len, auth_tag, auth_tag_len);
 254         } else {
 255                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 256                 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
 257                                         aad_len, auth_tag, auth_tag_len);
 258         }
 259 }
 260 #endif
 261
 262 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
 263                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 264                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 265                         u8 *auth_tag, unsigned long auth_tag_len);
 266
 267 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
 268                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 269                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 270                         u8 *auth_tag, unsigned long auth_tag_len);
 271
 272 static inline struct
 273 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 274 {
 275         unsigned long align = AESNI_ALIGN;
 276
 277         if (align <= crypto_tfm_ctx_alignment())
 278                 align = 1;
 279         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 280 }
 281
 282 static inline struct
 283 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 284 {
 285         unsigned long align = AESNI_ALIGN;
 286
 287         if (align <= crypto_tfm_ctx_alignment())
 288                 align = 1;
 289         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 290 }
 291 #endif
 292
 293 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 294 {
 295         unsigned long addr = (unsigned long)raw_ctx;
 296         unsigned long align = AESNI_ALIGN;
 297
 298         if (align <= crypto_tfm_ctx_alignment())
 299                 align = 1;
 300         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 301 }
 302
 303 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 304                               const u8 *in_key, unsigned int key_len)
 305 {
 306         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 307         u32 *flags = &tfm->crt_flags;
 308         int err;
 309
 310         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 311             key_len != AES_KEYSIZE_256) {
 312                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 313                 return -EINVAL;
 314         }
 315
 316         if (!irq_fpu_usable())
 317                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 318         else {
 319                 kernel_fpu_begin();
 320                 err = aesni_set_key(ctx, in_key, key_len);
 321                 kernel_fpu_end();
 322         }
 323
 324         return err;
 325 }
 326
 327 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 328                        unsigned int key_len)
 329 {
 330         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 331 }
 332
 333 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 334 {
 335         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 336
 337         if (!irq_fpu_usable())
 338                 crypto_aes_encrypt_x86(ctx, dst, src);
 339         else {
 340                 kernel_fpu_begin();
 341                 aesni_enc(ctx, dst, src);
 342                 kernel_fpu_end();
 343         }
 344 }
 345
 346 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 347 {
 348         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 349
 350         if (!irq_fpu_usable())
 351                 crypto_aes_decrypt_x86(ctx, dst, src);
 352         else {
 353                 kernel_fpu_begin();
 354                 aesni_dec(ctx, dst, src);
 355                 kernel_fpu_end();
 356         }
 357 }
 358
 359 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 360 {
 361         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 362
 363         aesni_enc(ctx, dst, src);
 364 }
 365
 366 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 367 {
 368         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 369
 370         aesni_dec(ctx, dst, src);
 371 }
 372
 373 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 374                                  unsigned int len)
 375 {
 376         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 377                                   crypto_skcipher_ctx(tfm), key, len);
 378 }
 379
 380 static int ecb_encrypt(struct skcipher_request *req)
 381 {
 382         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 383         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 384         struct skcipher_walk walk;
 385         unsigned int nbytes;
 386         int err;
 387
 388         err = skcipher_walk_virt(&walk, req, true);
 389
 390         kernel_fpu_begin();
 391         while ((nbytes = walk.nbytes)) {
 392                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 393                               nbytes & AES_BLOCK_MASK);
 394                 nbytes &= AES_BLOCK_SIZE - 1;
 395                 err = skcipher_walk_done(&walk, nbytes);
 396         }
 397         kernel_fpu_end();
 398
 399         return err;
 400 }
 401
 402 static int ecb_decrypt(struct skcipher_request *req)
 403 {
 404         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 405         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 406         struct skcipher_walk walk;
 407         unsigned int nbytes;
 408         int err;
 409
 410         err = skcipher_walk_virt(&walk, req, true);
 411
 412         kernel_fpu_begin();
 413         while ((nbytes = walk.nbytes)) {
 414                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 415                               nbytes & AES_BLOCK_MASK);
 416                 nbytes &= AES_BLOCK_SIZE - 1;
 417                 err = skcipher_walk_done(&walk, nbytes);
 418         }
 419         kernel_fpu_end();
 420
 421         return err;
 422 }
 423
 424 static int cbc_encrypt(struct skcipher_request *req)
 425 {
 426         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 427         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 428         struct skcipher_walk walk;
 429         unsigned int nbytes;
 430         int err;
 431
 432         err = skcipher_walk_virt(&walk, req, true);
 433
 434         kernel_fpu_begin();
 435         while ((nbytes = walk.nbytes)) {
 436                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 437                               nbytes & AES_BLOCK_MASK, walk.iv);
 438                 nbytes &= AES_BLOCK_SIZE - 1;
 439                 err = skcipher_walk_done(&walk, nbytes);
 440         }
 441         kernel_fpu_end();
 442
 443         return err;
 444 }
 445
 446 static int cbc_decrypt(struct skcipher_request *req)
 447 {
 448         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 449         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 450         struct skcipher_walk walk;
 451         unsigned int nbytes;
 452         int err;
 453
 454         err = skcipher_walk_virt(&walk, req, true);
 455
 456         kernel_fpu_begin();
 457         while ((nbytes = walk.nbytes)) {
 458                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 459                               nbytes & AES_BLOCK_MASK, walk.iv);
 460                 nbytes &= AES_BLOCK_SIZE - 1;
 461                 err = skcipher_walk_done(&walk, nbytes);
 462         }
 463         kernel_fpu_end();
 464
 465         return err;
 466 }
 467
 468 #ifdef CONFIG_X86_64
 469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 470                             struct skcipher_walk *walk)
 471 {
 472         u8 *ctrblk = walk->iv;
 473         u8 keystream[AES_BLOCK_SIZE];
 474         u8 *src = walk->src.virt.addr;
 475         u8 *dst = walk->dst.virt.addr;
 476         unsigned int nbytes = walk->nbytes;
 477
 478         aesni_enc(ctx, keystream, ctrblk);
 479         crypto_xor_cpy(dst, keystream, src, nbytes);
 480
 481         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 482 }
 483
 484 #ifdef CONFIG_AS_AVX
 485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 486                               const u8 *in, unsigned int len, u8 *iv)
 487 {
 488         /*
 489          * based on key length, override with the by8 version
 490          * of ctr mode encryption/decryption for improved performance
 491          * aes_set_key_common() ensures that key length is one of
 492          * {128,192,256}
 493          */
 494         if (ctx->key_length == AES_KEYSIZE_128)
 495                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 496         else if (ctx->key_length == AES_KEYSIZE_192)
 497                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 498         else
 499                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 500 }
 501 #endif
 502
 503 static int ctr_crypt(struct skcipher_request *req)
 504 {
 505         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 506         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 507         struct skcipher_walk walk;
 508         unsigned int nbytes;
 509         int err;
 510
 511         err = skcipher_walk_virt(&walk, req, true);
 512
 513         kernel_fpu_begin();
 514         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 515                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 516                                       nbytes & AES_BLOCK_MASK, walk.iv);
 517                 nbytes &= AES_BLOCK_SIZE - 1;
 518                 err = skcipher_walk_done(&walk, nbytes);
 519         }
 520         if (walk.nbytes) {
 521                 ctr_crypt_final(ctx, &walk);
 522                 err = skcipher_walk_done(&walk, 0);
 523         }
 524         kernel_fpu_end();
 525
 526         return err;
 527 }
 528
 529 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 530                             unsigned int keylen)
 531 {
 532         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 533         int err;
 534
 535         err = xts_verify_key(tfm, key, keylen);
 536         if (err)
 537                 return err;
 538
 539         keylen /= 2;
 540
 541         /* first half of xts-key is for crypt */
 542         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 543                                  key, keylen);
 544         if (err)
 545                 return err;
 546
 547         /* second half of xts-key is for tweak */
 548         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 549                                   key + keylen, keylen);
 550 }
 551
 552
 553 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 554 {
 555         aesni_enc(ctx, out, in);
 556 }
 557
 558 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 559 {
 560         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
 561 }
 562
 563 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 564 {
 565         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
 566 }
 567
 568 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 569 {
 570         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
 571 }
 572
 573 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 574 {
 575         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
 576 }
 577
 578 static const struct common_glue_ctx aesni_enc_xts = {
 579         .num_funcs = 2,
 580         .fpu_blocks_limit = 1,
 581
 582         .funcs = { {
 583                 .num_blocks = 8,
 584                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
 585         }, {
 586                 .num_blocks = 1,
 587                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
 588         } }
 589 };
 590
 591 static const struct common_glue_ctx aesni_dec_xts = {
 592         .num_funcs = 2,
 593         .fpu_blocks_limit = 1,
 594
 595         .funcs = { {
 596                 .num_blocks = 8,
 597                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
 598         }, {
 599                 .num_blocks = 1,
 600                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
 601         } }
 602 };
 603
 604 static int xts_encrypt(struct skcipher_request *req)
 605 {
 606         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 607         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 608
 609         return glue_xts_req_128bit(&aesni_enc_xts, req,
 610                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 611                                    aes_ctx(ctx->raw_tweak_ctx),
 612                                    aes_ctx(ctx->raw_crypt_ctx));
 613 }
 614
 615 static int xts_decrypt(struct skcipher_request *req)
 616 {
 617         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 618         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 619
 620         return glue_xts_req_128bit(&aesni_dec_xts, req,
 621                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 622                                    aes_ctx(ctx->raw_tweak_ctx),
 623                                    aes_ctx(ctx->raw_crypt_ctx));
 624 }
 625
 626 static int rfc4106_init(struct crypto_aead *aead)
 627 {
 628         struct cryptd_aead *cryptd_tfm;
 629         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 630
 631         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
 632                                        CRYPTO_ALG_INTERNAL,
 633                                        CRYPTO_ALG_INTERNAL);
 634         if (IS_ERR(cryptd_tfm))
 635                 return PTR_ERR(cryptd_tfm);
 636
 637         *ctx = cryptd_tfm;
 638         crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
 639         return 0;
 640 }
 641
 642 static void rfc4106_exit(struct crypto_aead *aead)
 643 {
 644         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 645
 646         cryptd_free_aead(*ctx);
 647 }
 648
 649 static int
 650 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 651 {
 652         struct crypto_cipher *tfm;
 653         int ret;
 654
 655         tfm = crypto_alloc_cipher("aes", 0, 0);
 656         if (IS_ERR(tfm))
 657                 return PTR_ERR(tfm);
 658
 659         ret = crypto_cipher_setkey(tfm, key, key_len);
 660         if (ret)
 661                 goto out_free_cipher;
 662
 663         /* Clear the data in the hash sub key container to zero.*/
 664         /* We want to cipher all zeros to create the hash sub key. */
 665         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 666
 667         crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
 668
 669 out_free_cipher:
 670         crypto_free_cipher(tfm);
 671         return ret;
 672 }
 673
 674 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 675                                   unsigned int key_len)
 676 {
 677         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 678
 679         if (key_len < 4) {
 680                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 681                 return -EINVAL;
 682         }
 683         /*Account for 4 byte nonce at the end.*/
 684         key_len -= 4;
 685
 686         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 687
 688         return aes_set_key_common(crypto_aead_tfm(aead),
 689                                   &ctx->aes_key_expanded, key, key_len) ?:
 690                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 691 }
 692
 693 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
 694                                   unsigned int key_len)
 695 {
 696         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 697         struct cryptd_aead *cryptd_tfm = *ctx;
 698
 699         return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
 700 }
 701
 702 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 703                                        unsigned int authsize)
 704 {
 705         switch (authsize) {
 706         case 8:
 707         case 12:
 708         case 16:
 709                 break;
 710         default:
 711                 return -EINVAL;
 712         }
 713
 714         return 0;
 715 }
 716
 717 /* This is the Integrity Check Value (aka the authentication tag length and can
 718  * be 8, 12 or 16 bytes long. */
 719 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
 720                                        unsigned int authsize)
 721 {
 722         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 723         struct cryptd_aead *cryptd_tfm = *ctx;
 724
 725         return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
 726 }
 727
 728 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 729                                        unsigned int authsize)
 730 {
 731         switch (authsize) {
 732         case 4:
 733         case 8:
 734         case 12:
 735         case 13:
 736         case 14:
 737         case 15:
 738         case 16:
 739                 break;
 740         default:
 741                 return -EINVAL;
 742         }
 743
 744         return 0;
 745 }
 746
 747 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 748                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 749 {
 750         u8 one_entry_in_sg = 0;
 751         u8 *src, *dst, *assoc;
 752         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 753         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 754         struct scatter_walk src_sg_walk;
 755         struct scatter_walk dst_sg_walk = {};
 756
 757         if (sg_is_last(req->src) &&
 758             (!PageHighMem(sg_page(req->src)) ||
 759             req->src->offset + req->src->length <= PAGE_SIZE) &&
 760             sg_is_last(req->dst) &&
 761             (!PageHighMem(sg_page(req->dst)) ||
 762             req->dst->offset + req->dst->length <= PAGE_SIZE)) {
 763                 one_entry_in_sg = 1;
 764                 scatterwalk_start(&src_sg_walk, req->src);
 765                 assoc = scatterwalk_map(&src_sg_walk);
 766                 src = assoc + req->assoclen;
 767                 dst = src;
 768                 if (unlikely(req->src != req->dst)) {
 769                         scatterwalk_start(&dst_sg_walk, req->dst);
 770                         dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
 771                 }
 772         } else {
 773                 /* Allocate memory for src, dst, assoc */
 774                 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
 775                         GFP_ATOMIC);
 776                 if (unlikely(!assoc))
 777                         return -ENOMEM;
 778                 scatterwalk_map_and_copy(assoc, req->src, 0,
 779                                          req->assoclen + req->cryptlen, 0);
 780                 src = assoc + req->assoclen;
 781                 dst = src;
 782         }
 783
 784         kernel_fpu_begin();
 785         aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
 786                           hash_subkey, assoc, assoclen,
 787                           dst + req->cryptlen, auth_tag_len);
 788         kernel_fpu_end();
 789
 790         /* The authTag (aka the Integrity Check Value) needs to be written
 791          * back to the packet. */
 792         if (one_entry_in_sg) {
 793                 if (unlikely(req->src != req->dst)) {
 794                         scatterwalk_unmap(dst - req->assoclen);
 795                         scatterwalk_advance(&dst_sg_walk, req->dst->length);
 796                         scatterwalk_done(&dst_sg_walk, 1, 0);
 797                 }
 798                 scatterwalk_unmap(assoc);
 799                 scatterwalk_advance(&src_sg_walk, req->src->length);
 800                 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
 801         } else {
 802                 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
 803                                          req->cryptlen + auth_tag_len, 1);
 804                 kfree(assoc);
 805         }
 806         return 0;
 807 }
 808
 809 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 810                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 811 {
 812         u8 one_entry_in_sg = 0;
 813         u8 *src, *dst, *assoc;
 814         unsigned long tempCipherLen = 0;
 815         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 816         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 817         u8 authTag[16];
 818         struct scatter_walk src_sg_walk;
 819         struct scatter_walk dst_sg_walk = {};
 820         int retval = 0;
 821
 822         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
 823
 824         if (sg_is_last(req->src) &&
 825             (!PageHighMem(sg_page(req->src)) ||
 826             req->src->offset + req->src->length <= PAGE_SIZE) &&
 827             sg_is_last(req->dst) && req->dst->length &&
 828             (!PageHighMem(sg_page(req->dst)) ||
 829             req->dst->offset + req->dst->length <= PAGE_SIZE)) {
 830                 one_entry_in_sg = 1;
 831                 scatterwalk_start(&src_sg_walk, req->src);
 832                 assoc = scatterwalk_map(&src_sg_walk);
 833                 src = assoc + req->assoclen;
 834                 dst = src;
 835                 if (unlikely(req->src != req->dst)) {
 836                         scatterwalk_start(&dst_sg_walk, req->dst);
 837                         dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
 838                 }
 839         } else {
 840                 /* Allocate memory for src, dst, assoc */
 841                 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
 842                 if (!assoc)
 843                         return -ENOMEM;
 844                 scatterwalk_map_and_copy(assoc, req->src, 0,
 845                                          req->assoclen + req->cryptlen, 0);
 846                 src = assoc + req->assoclen;
 847                 dst = src;
 848         }
 849
 850
 851         kernel_fpu_begin();
 852         aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
 853                           hash_subkey, assoc, assoclen,
 854                           authTag, auth_tag_len);
 855         kernel_fpu_end();
 856
 857         /* Compare generated tag with passed in tag. */
 858         retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
 859                 -EBADMSG : 0;
 860
 861         if (one_entry_in_sg) {
 862                 if (unlikely(req->src != req->dst)) {
 863                         scatterwalk_unmap(dst - req->assoclen);
 864                         scatterwalk_advance(&dst_sg_walk, req->dst->length);
 865                         scatterwalk_done(&dst_sg_walk, 1, 0);
 866                 }
 867                 scatterwalk_unmap(assoc);
 868                 scatterwalk_advance(&src_sg_walk, req->src->length);
 869                 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
 870         } else {
 871                 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
 872                                          tempCipherLen, 1);
 873                 kfree(assoc);
 874         }
 875         return retval;
 876
 877 }
 878
 879 static int helper_rfc4106_encrypt(struct aead_request *req)
 880 {
 881         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 882         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 883         void *aes_ctx = &(ctx->aes_key_expanded);
 884         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 885         unsigned int i;
 886         __be32 counter = cpu_to_be32(1);
 887
 888         /* Assuming we are supporting rfc4106 64-bit extended */
 889         /* sequence numbers We need to have the AAD length equal */
 890         /* to 16 or 20 bytes */
 891         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 892                 return -EINVAL;
 893
 894         /* IV below built */
 895         for (i = 0; i < 4; i++)
 896                 *(iv+i) = ctx->nonce[i];
 897         for (i = 0; i < 8; i++)
 898                 *(iv+4+i) = req->iv[i];
 899         *((__be32 *)(iv+12)) = counter;
 900
 901         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 902                               aes_ctx);
 903 }
 904
 905 static int helper_rfc4106_decrypt(struct aead_request *req)
 906 {
 907         __be32 counter = cpu_to_be32(1);
 908         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 909         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 910         void *aes_ctx = &(ctx->aes_key_expanded);
 911         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 912         unsigned int i;
 913
 914         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 915                 return -EINVAL;
 916
 917         /* Assuming we are supporting rfc4106 64-bit extended */
 918         /* sequence numbers We need to have the AAD length */
 919         /* equal to 16 or 20 bytes */
 920
 921         /* IV below built */
 922         for (i = 0; i < 4; i++)
 923                 *(iv+i) = ctx->nonce[i];
 924         for (i = 0; i < 8; i++)
 925                 *(iv+4+i) = req->iv[i];
 926         *((__be32 *)(iv+12)) = counter;
 927
 928         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 929                               aes_ctx);
 930 }
 931
 932 static int gcmaes_wrapper_encrypt(struct aead_request *req)
 933 {
 934         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 935         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 936         struct cryptd_aead *cryptd_tfm = *ctx;
 937
 938         tfm = &cryptd_tfm->base;
 939         if (irq_fpu_usable() && (!in_atomic() ||
 940                                  !cryptd_aead_queued(cryptd_tfm)))
 941                 tfm = cryptd_aead_child(cryptd_tfm);
 942
 943         aead_request_set_tfm(req, tfm);
 944
 945         return crypto_aead_encrypt(req);
 946 }
 947
 948 static int gcmaes_wrapper_decrypt(struct aead_request *req)
 949 {
 950         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 951         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 952         struct cryptd_aead *cryptd_tfm = *ctx;
 953
 954         tfm = &cryptd_tfm->base;
 955         if (irq_fpu_usable() && (!in_atomic() ||
 956                                  !cryptd_aead_queued(cryptd_tfm)))
 957                 tfm = cryptd_aead_child(cryptd_tfm);
 958
 959         aead_request_set_tfm(req, tfm);
 960
 961         return crypto_aead_decrypt(req);
 962 }
 963 #endif
 964
 965 static struct crypto_alg aesni_algs[] = { {
 966         .cra_name               = "aes",
 967         .cra_driver_name        = "aes-aesni",
 968         .cra_priority           = 300,
 969         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 970         .cra_blocksize          = AES_BLOCK_SIZE,
 971         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 972         .cra_module             = THIS_MODULE,
 973         .cra_u  = {
 974                 .cipher = {
 975                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 976                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 977                         .cia_setkey             = aes_set_key,
 978                         .cia_encrypt            = aes_encrypt,
 979                         .cia_decrypt            = aes_decrypt
 980                 }
 981         }
 982 }, {
 983         .cra_name               = "__aes",
 984         .cra_driver_name        = "__aes-aesni",
 985         .cra_priority           = 300,
 986         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
 987         .cra_blocksize          = AES_BLOCK_SIZE,
 988         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 989         .cra_module             = THIS_MODULE,
 990         .cra_u  = {
 991                 .cipher = {
 992                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 993                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 994                         .cia_setkey             = aes_set_key,
 995                         .cia_encrypt            = __aes_encrypt,
 996                         .cia_decrypt            = __aes_decrypt
 997                 }
 998         }
 999 } };
1000
1001 static struct skcipher_alg aesni_skciphers[] = {
1002         {
1003                 .base = {
1004                         .cra_name               = "__ecb(aes)",
1005                         .cra_driver_name        = "__ecb-aes-aesni",
1006                         .cra_priority           = 400,
1007                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1008                         .cra_blocksize          = AES_BLOCK_SIZE,
1009                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1010                         .cra_module             = THIS_MODULE,
1011                 },
1012                 .min_keysize    = AES_MIN_KEY_SIZE,
1013                 .max_keysize    = AES_MAX_KEY_SIZE,
1014                 .setkey         = aesni_skcipher_setkey,
1015                 .encrypt        = ecb_encrypt,
1016                 .decrypt        = ecb_decrypt,
1017         }, {
1018                 .base = {
1019                         .cra_name               = "__cbc(aes)",
1020                         .cra_driver_name        = "__cbc-aes-aesni",
1021                         .cra_priority           = 400,
1022                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1023                         .cra_blocksize          = AES_BLOCK_SIZE,
1024                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1025                         .cra_module             = THIS_MODULE,
1026                 },
1027                 .min_keysize    = AES_MIN_KEY_SIZE,
1028                 .max_keysize    = AES_MAX_KEY_SIZE,
1029                 .ivsize         = AES_BLOCK_SIZE,
1030                 .setkey         = aesni_skcipher_setkey,
1031                 .encrypt        = cbc_encrypt,
1032                 .decrypt        = cbc_decrypt,
1033 #ifdef CONFIG_X86_64
1034         }, {
1035                 .base = {
1036                         .cra_name               = "__ctr(aes)",
1037                         .cra_driver_name        = "__ctr-aes-aesni",
1038                         .cra_priority           = 400,
1039                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1040                         .cra_blocksize          = 1,
1041                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1042                         .cra_module             = THIS_MODULE,
1043                 },
1044                 .min_keysize    = AES_MIN_KEY_SIZE,
1045                 .max_keysize    = AES_MAX_KEY_SIZE,
1046                 .ivsize         = AES_BLOCK_SIZE,
1047                 .chunksize      = AES_BLOCK_SIZE,
1048                 .setkey         = aesni_skcipher_setkey,
1049                 .encrypt        = ctr_crypt,
1050                 .decrypt        = ctr_crypt,
1051         }, {
1052                 .base = {
1053                         .cra_name               = "__xts(aes)",
1054                         .cra_driver_name        = "__xts-aes-aesni",
1055                         .cra_priority           = 401,
1056                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1057                         .cra_blocksize          = AES_BLOCK_SIZE,
1058                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1059                         .cra_module             = THIS_MODULE,
1060                 },
1061                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1062                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1063                 .ivsize         = AES_BLOCK_SIZE,
1064                 .setkey         = xts_aesni_setkey,
1065                 .encrypt        = xts_encrypt,
1066                 .decrypt        = xts_decrypt,
1067 #endif
1068         }
1069 };
1070
1071 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1072
1073 struct {
1074         const char *algname;
1075         const char *drvname;
1076         const char *basename;
1077         struct simd_skcipher_alg *simd;
1078 } aesni_simd_skciphers2[] = {
1079 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
1080     IS_BUILTIN(CONFIG_CRYPTO_PCBC)
1081         {
1082                 .algname        = "pcbc(aes)",
1083                 .drvname        = "pcbc-aes-aesni",
1084                 .basename       = "fpu(pcbc(__aes-aesni))",
1085         },
1086 #endif
1087 };
1088
1089 #ifdef CONFIG_X86_64
1090 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1091                                   unsigned int key_len)
1092 {
1093         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1094
1095         return aes_set_key_common(crypto_aead_tfm(aead),
1096                                   &ctx->aes_key_expanded, key, key_len) ?:
1097                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1098 }
1099
1100 static int generic_gcmaes_encrypt(struct aead_request *req)
1101 {
1102         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1103         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1104         void *aes_ctx = &(ctx->aes_key_expanded);
1105         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1106         __be32 counter = cpu_to_be32(1);
1107
1108         memcpy(iv, req->iv, 12);
1109         *((__be32 *)(iv+12)) = counter;
1110
1111         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1112                               aes_ctx);
1113 }
1114
1115 static int generic_gcmaes_decrypt(struct aead_request *req)
1116 {
1117         __be32 counter = cpu_to_be32(1);
1118         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1119         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1120         void *aes_ctx = &(ctx->aes_key_expanded);
1121         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1122
1123         memcpy(iv, req->iv, 12);
1124         *((__be32 *)(iv+12)) = counter;
1125
1126         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1127                               aes_ctx);
1128 }
1129
1130 static int generic_gcmaes_init(struct crypto_aead *aead)
1131 {
1132         struct cryptd_aead *cryptd_tfm;
1133         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1134
1135         cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
1136                                        CRYPTO_ALG_INTERNAL,
1137                                        CRYPTO_ALG_INTERNAL);
1138         if (IS_ERR(cryptd_tfm))
1139                 return PTR_ERR(cryptd_tfm);
1140
1141         *ctx = cryptd_tfm;
1142         crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
1143
1144         return 0;
1145 }
1146
1147 static void generic_gcmaes_exit(struct crypto_aead *aead)
1148 {
1149         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1150
1151         cryptd_free_aead(*ctx);
1152 }
1153
1154 static struct aead_alg aesni_aead_algs[] = { {
1155         .setkey                 = common_rfc4106_set_key,
1156         .setauthsize            = common_rfc4106_set_authsize,
1157         .encrypt                = helper_rfc4106_encrypt,
1158         .decrypt                = helper_rfc4106_decrypt,
1159         .ivsize                 = GCM_RFC4106_IV_SIZE,
1160         .maxauthsize            = 16,
1161         .base = {
1162                 .cra_name               = "__gcm-aes-aesni",
1163                 .cra_driver_name        = "__driver-gcm-aes-aesni",
1164                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1165                 .cra_blocksize          = 1,
1166                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1167                 .cra_alignmask          = AESNI_ALIGN - 1,
1168                 .cra_module             = THIS_MODULE,
1169         },
1170 }, {
1171         .init                   = rfc4106_init,
1172         .exit                   = rfc4106_exit,
1173         .setkey                 = gcmaes_wrapper_set_key,
1174         .setauthsize            = gcmaes_wrapper_set_authsize,
1175         .encrypt                = gcmaes_wrapper_encrypt,
1176         .decrypt                = gcmaes_wrapper_decrypt,
1177         .ivsize                 = GCM_RFC4106_IV_SIZE,
1178         .maxauthsize            = 16,
1179         .base = {
1180                 .cra_name               = "rfc4106(gcm(aes))",
1181                 .cra_driver_name        = "rfc4106-gcm-aesni",
1182                 .cra_priority           = 400,
1183                 .cra_flags              = CRYPTO_ALG_ASYNC,
1184                 .cra_blocksize          = 1,
1185                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
1186                 .cra_module             = THIS_MODULE,
1187         },
1188 }, {
1189         .setkey                 = generic_gcmaes_set_key,
1190         .setauthsize            = generic_gcmaes_set_authsize,
1191         .encrypt                = generic_gcmaes_encrypt,
1192         .decrypt                = generic_gcmaes_decrypt,
1193         .ivsize                 = GCM_AES_IV_SIZE,
1194         .maxauthsize            = 16,
1195         .base = {
1196                 .cra_name               = "__generic-gcm-aes-aesni",
1197                 .cra_driver_name        = "__driver-generic-gcm-aes-aesni",
1198                 .cra_priority           = 0,
1199                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1200                 .cra_blocksize          = 1,
1201                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1202                 .cra_alignmask          = AESNI_ALIGN - 1,
1203                 .cra_module             = THIS_MODULE,
1204         },
1205 }, {
1206         .init                   = generic_gcmaes_init,
1207         .exit                   = generic_gcmaes_exit,
1208         .setkey                 = gcmaes_wrapper_set_key,
1209         .setauthsize            = gcmaes_wrapper_set_authsize,
1210         .encrypt                = gcmaes_wrapper_encrypt,
1211         .decrypt                = gcmaes_wrapper_decrypt,
1212         .ivsize                 = GCM_AES_IV_SIZE,
1213         .maxauthsize            = 16,
1214         .base = {
1215                 .cra_name               = "gcm(aes)",
1216                 .cra_driver_name        = "generic-gcm-aesni",
1217                 .cra_priority           = 400,
1218                 .cra_flags              = CRYPTO_ALG_ASYNC,
1219                 .cra_blocksize          = 1,
1220                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
1221                 .cra_module             = THIS_MODULE,
1222         },
1223 } };
1224 #else
1225 static struct aead_alg aesni_aead_algs[0];
1226 #endif
1227
1228
1229 static const struct x86_cpu_id aesni_cpu_id[] = {
1230         X86_FEATURE_MATCH(X86_FEATURE_AES),
1231         {}
1232 };
1233 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1234
1235 static void aesni_free_simds(void)
1236 {
1237         int i;
1238
1239         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1240                     aesni_simd_skciphers[i]; i++)
1241                 simd_skcipher_free(aesni_simd_skciphers[i]);
1242
1243         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
1244                 if (aesni_simd_skciphers2[i].simd)
1245                         simd_skcipher_free(aesni_simd_skciphers2[i].simd);
1246 }
1247
1248 static int __init aesni_init(void)
1249 {
1250         struct simd_skcipher_alg *simd;
1251         const char *basename;
1252         const char *algname;
1253         const char *drvname;
1254         int err;
1255         int i;
1256
1257         if (!x86_match_cpu(aesni_cpu_id))
1258                 return -ENODEV;
1259 #ifdef CONFIG_X86_64
1260 #ifdef CONFIG_AS_AVX2
1261         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1262                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1263                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1264                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1265         } else
1266 #endif
1267 #ifdef CONFIG_AS_AVX
1268         if (boot_cpu_has(X86_FEATURE_AVX)) {
1269                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1270                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1271                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1272         } else
1273 #endif
1274         {
1275                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1276                 aesni_gcm_enc_tfm = aesni_gcm_enc;
1277                 aesni_gcm_dec_tfm = aesni_gcm_dec;
1278         }
1279         aesni_ctr_enc_tfm = aesni_ctr_enc;
1280 #ifdef CONFIG_AS_AVX
1281         if (boot_cpu_has(X86_FEATURE_AVX)) {
1282                 /* optimize performance of ctr mode encryption transform */
1283                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1284                 pr_info("AES CTR mode by8 optimization enabled\n");
1285         }
1286 #endif
1287 #endif
1288
1289         err = crypto_fpu_init();
1290         if (err)
1291                 return err;
1292
1293         err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1294         if (err)
1295                 goto fpu_exit;
1296
1297         err = crypto_register_skciphers(aesni_skciphers,
1298                                         ARRAY_SIZE(aesni_skciphers));
1299         if (err)
1300                 goto unregister_algs;
1301
1302         err = crypto_register_aeads(aesni_aead_algs,
1303                                     ARRAY_SIZE(aesni_aead_algs));
1304         if (err)
1305                 goto unregister_skciphers;
1306
1307         for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1308                 algname = aesni_skciphers[i].base.cra_name + 2;
1309                 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1310                 basename = aesni_skciphers[i].base.cra_driver_name;
1311                 simd = simd_skcipher_create_compat(algname, drvname, basename);
1312                 err = PTR_ERR(simd);
1313                 if (IS_ERR(simd))
1314                         goto unregister_simds;
1315
1316                 aesni_simd_skciphers[i] = simd;
1317         }
1318
1319         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
1320                 algname = aesni_simd_skciphers2[i].algname;
1321                 drvname = aesni_simd_skciphers2[i].drvname;
1322                 basename = aesni_simd_skciphers2[i].basename;
1323                 simd = simd_skcipher_create_compat(algname, drvname, basename);
1324                 err = PTR_ERR(simd);
1325                 if (IS_ERR(simd))
1326                         continue;
1327
1328                 aesni_simd_skciphers2[i].simd = simd;
1329         }
1330
1331         return 0;
1332
1333 unregister_simds:
1334         aesni_free_simds();
1335         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1336 unregister_skciphers:
1337         crypto_unregister_skciphers(aesni_skciphers,
1338                                     ARRAY_SIZE(aesni_skciphers));
1339 unregister_algs:
1340         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1341 fpu_exit:
1342         crypto_fpu_exit();
1343         return err;
1344 }
1345
1346 static void __exit aesni_exit(void)
1347 {
1348         aesni_free_simds();
1349         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1350         crypto_unregister_skciphers(aesni_skciphers,
1351                                     ARRAY_SIZE(aesni_skciphers));
1352         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1353
1354         crypto_fpu_exit();
1355 }
1356
1357 late_initcall(aesni_init);
1358 module_exit(aesni_exit);
1359
1360 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1361 MODULE_LICENSE("GPL");
1362 MODULE_ALIAS_CRYPTO("aes");