x86/crypto/aesni-intel_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Support for Intel AES-NI instructions. This file contains glue
   4  * code, the real AES implementation is in intel-aes_asm.S.
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/types.h>
  20 #include <linux/module.h>
  21 #include <linux/err.h>
  22 #include <crypto/algapi.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/b128ops.h>
  26 #include <crypto/gcm.h>
  27 #include <crypto/xts.h>
  28 #include <asm/cpu_device_id.h>
  29 #include <asm/simd.h>
  30 #include <crypto/scatterwalk.h>
  31 #include <crypto/internal/aead.h>
  32 #include <crypto/internal/simd.h>
  33 #include <crypto/internal/skcipher.h>
  34 #include <linux/jump_label.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/static_call.h>
  38
  39
  40 #define AESNI_ALIGN     16
  41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  42 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  43 #define RFC4106_HASH_SUBKEY_SIZE 16
  44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  47
  48 /* This data is stored at the end of the crypto_tfm struct.
  49  * It's a type of per "session" data storage location.
  50  * This needs to be 16 byte aligned.
  51  */
  52 struct aesni_rfc4106_gcm_ctx {
  53         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  54         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  55         u8 nonce[4];
  56 };
  57
  58 struct generic_gcmaes_ctx {
  59         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  60         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  61 };
  62
  63 struct aesni_xts_ctx {
  64         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  65         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  66 };
  67
  68 #define GCM_BLOCK_LEN 16
  69
  70 struct gcm_context_data {
  71         /* init, update and finalize context data */
  72         u8 aad_hash[GCM_BLOCK_LEN];
  73         u64 aad_length;
  74         u64 in_length;
  75         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  76         u8 orig_IV[GCM_BLOCK_LEN];
  77         u8 current_counter[GCM_BLOCK_LEN];
  78         u64 partial_block_len;
  79         u64 unused;
  80         u8 hash_keys[GCM_BLOCK_LEN * 16];
  81 };
  82
  83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  84                              unsigned int key_len);
  85 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
  86 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
  87 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  88                               const u8 *in, unsigned int len);
  89 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  90                               const u8 *in, unsigned int len);
  91 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len, u8 *iv);
  93 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len, u8 *iv);
  95 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  96                                   const u8 *in, unsigned int len, u8 *iv);
  97 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  98                                   const u8 *in, unsigned int len, u8 *iv);
  99
 100 #define AVX_GEN2_OPTSIZE 640
 101 #define AVX_GEN4_OPTSIZE 4096
 102
 103 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 104                                   const u8 *in, unsigned int len, u8 *iv);
 105
 106 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 107                                   const u8 *in, unsigned int len, u8 *iv);
 108
 109 #ifdef CONFIG_X86_64
 110
 111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 112                               const u8 *in, unsigned int len, u8 *iv);
 113 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
 114
 115 /* Scatter / Gather routines, with args similar to above */
 116 asmlinkage void aesni_gcm_init(void *ctx,
 117                                struct gcm_context_data *gdata,
 118                                u8 *iv,
 119                                u8 *hash_subkey, const u8 *aad,
 120                                unsigned long aad_len);
 121 asmlinkage void aesni_gcm_enc_update(void *ctx,
 122                                      struct gcm_context_data *gdata, u8 *out,
 123                                      const u8 *in, unsigned long plaintext_len);
 124 asmlinkage void aesni_gcm_dec_update(void *ctx,
 125                                      struct gcm_context_data *gdata, u8 *out,
 126                                      const u8 *in,
 127                                      unsigned long ciphertext_len);
 128 asmlinkage void aesni_gcm_finalize(void *ctx,
 129                                    struct gcm_context_data *gdata,
 130                                    u8 *auth_tag, unsigned long auth_tag_len);
 131
 132 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 133                 void *keys, u8 *out, unsigned int num_bytes);
 134 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 135                 void *keys, u8 *out, unsigned int num_bytes);
 136 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 137                 void *keys, u8 *out, unsigned int num_bytes);
 138
 139
 140 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
 141         const void *keys, u8 *out, unsigned int num_bytes,
 142         unsigned int byte_ctr);
 143
 144 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
 145         const void *keys, u8 *out, unsigned int num_bytes,
 146         unsigned int byte_ctr);
 147
 148 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
 149         const void *keys, u8 *out, unsigned int num_bytes,
 150         unsigned int byte_ctr);
 151
 152 /*
 153  * asmlinkage void aesni_gcm_init_avx_gen2()
 154  * gcm_data *my_ctx_data, context data
 155  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 156  */
 157 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 158                                         struct gcm_context_data *gdata,
 159                                         u8 *iv,
 160                                         u8 *hash_subkey,
 161                                         const u8 *aad,
 162                                         unsigned long aad_len);
 163
 164 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 165                                      struct gcm_context_data *gdata, u8 *out,
 166                                      const u8 *in, unsigned long plaintext_len);
 167 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 168                                      struct gcm_context_data *gdata, u8 *out,
 169                                      const u8 *in,
 170                                      unsigned long ciphertext_len);
 171 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 172                                    struct gcm_context_data *gdata,
 173                                    u8 *auth_tag, unsigned long auth_tag_len);
 174
 175 /*
 176  * asmlinkage void aesni_gcm_init_avx_gen4()
 177  * gcm_data *my_ctx_data, context data
 178  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 179  */
 180 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 181                                         struct gcm_context_data *gdata,
 182                                         u8 *iv,
 183                                         u8 *hash_subkey,
 184                                         const u8 *aad,
 185                                         unsigned long aad_len);
 186
 187 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 188                                      struct gcm_context_data *gdata, u8 *out,
 189                                      const u8 *in, unsigned long plaintext_len);
 190 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 191                                      struct gcm_context_data *gdata, u8 *out,
 192                                      const u8 *in,
 193                                      unsigned long ciphertext_len);
 194 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 195                                    struct gcm_context_data *gdata,
 196                                    u8 *auth_tag, unsigned long auth_tag_len);
 197
 198 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
 199 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
 200
 201 static inline struct
 202 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 203 {
 204         unsigned long align = AESNI_ALIGN;
 205
 206         if (align <= crypto_tfm_ctx_alignment())
 207                 align = 1;
 208         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 209 }
 210
 211 static inline struct
 212 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 213 {
 214         unsigned long align = AESNI_ALIGN;
 215
 216         if (align <= crypto_tfm_ctx_alignment())
 217                 align = 1;
 218         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 219 }
 220 #endif
 221
 222 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 223 {
 224         unsigned long addr = (unsigned long)raw_ctx;
 225         unsigned long align = AESNI_ALIGN;
 226
 227         if (align <= crypto_tfm_ctx_alignment())
 228                 align = 1;
 229         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 230 }
 231
 232 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 233                               const u8 *in_key, unsigned int key_len)
 234 {
 235         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 236         int err;
 237
 238         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 239             key_len != AES_KEYSIZE_256)
 240                 return -EINVAL;
 241
 242         if (!crypto_simd_usable())
 243                 err = aes_expandkey(ctx, in_key, key_len);
 244         else {
 245                 kernel_fpu_begin();
 246                 err = aesni_set_key(ctx, in_key, key_len);
 247                 kernel_fpu_end();
 248         }
 249
 250         return err;
 251 }
 252
 253 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 254                        unsigned int key_len)
 255 {
 256         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 257 }
 258
 259 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 260 {
 261         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 262
 263         if (!crypto_simd_usable()) {
 264                 aes_encrypt(ctx, dst, src);
 265         } else {
 266                 kernel_fpu_begin();
 267                 aesni_enc(ctx, dst, src);
 268                 kernel_fpu_end();
 269         }
 270 }
 271
 272 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 273 {
 274         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 275
 276         if (!crypto_simd_usable()) {
 277                 aes_decrypt(ctx, dst, src);
 278         } else {
 279                 kernel_fpu_begin();
 280                 aesni_dec(ctx, dst, src);
 281                 kernel_fpu_end();
 282         }
 283 }
 284
 285 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 286                                  unsigned int len)
 287 {
 288         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 289                                   crypto_skcipher_ctx(tfm), key, len);
 290 }
 291
 292 static int ecb_encrypt(struct skcipher_request *req)
 293 {
 294         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 295         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 296         struct skcipher_walk walk;
 297         unsigned int nbytes;
 298         int err;
 299
 300         err = skcipher_walk_virt(&walk, req, false);
 301
 302         while ((nbytes = walk.nbytes)) {
 303                 kernel_fpu_begin();
 304                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 305                               nbytes & AES_BLOCK_MASK);
 306                 kernel_fpu_end();
 307                 nbytes &= AES_BLOCK_SIZE - 1;
 308                 err = skcipher_walk_done(&walk, nbytes);
 309         }
 310
 311         return err;
 312 }
 313
 314 static int ecb_decrypt(struct skcipher_request *req)
 315 {
 316         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 317         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 318         struct skcipher_walk walk;
 319         unsigned int nbytes;
 320         int err;
 321
 322         err = skcipher_walk_virt(&walk, req, false);
 323
 324         while ((nbytes = walk.nbytes)) {
 325                 kernel_fpu_begin();
 326                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 327                               nbytes & AES_BLOCK_MASK);
 328                 kernel_fpu_end();
 329                 nbytes &= AES_BLOCK_SIZE - 1;
 330                 err = skcipher_walk_done(&walk, nbytes);
 331         }
 332
 333         return err;
 334 }
 335
 336 static int cbc_encrypt(struct skcipher_request *req)
 337 {
 338         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 339         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 340         struct skcipher_walk walk;
 341         unsigned int nbytes;
 342         int err;
 343
 344         err = skcipher_walk_virt(&walk, req, false);
 345
 346         while ((nbytes = walk.nbytes)) {
 347                 kernel_fpu_begin();
 348                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 349                               nbytes & AES_BLOCK_MASK, walk.iv);
 350                 kernel_fpu_end();
 351                 nbytes &= AES_BLOCK_SIZE - 1;
 352                 err = skcipher_walk_done(&walk, nbytes);
 353         }
 354
 355         return err;
 356 }
 357
 358 static int cbc_decrypt(struct skcipher_request *req)
 359 {
 360         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 361         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 362         struct skcipher_walk walk;
 363         unsigned int nbytes;
 364         int err;
 365
 366         err = skcipher_walk_virt(&walk, req, false);
 367
 368         while ((nbytes = walk.nbytes)) {
 369                 kernel_fpu_begin();
 370                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 371                               nbytes & AES_BLOCK_MASK, walk.iv);
 372                 kernel_fpu_end();
 373                 nbytes &= AES_BLOCK_SIZE - 1;
 374                 err = skcipher_walk_done(&walk, nbytes);
 375         }
 376
 377         return err;
 378 }
 379
 380 static int cts_cbc_encrypt(struct skcipher_request *req)
 381 {
 382         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 383         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 384         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 385         struct scatterlist *src = req->src, *dst = req->dst;
 386         struct scatterlist sg_src[2], sg_dst[2];
 387         struct skcipher_request subreq;
 388         struct skcipher_walk walk;
 389         int err;
 390
 391         skcipher_request_set_tfm(&subreq, tfm);
 392         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 393                                       NULL, NULL);
 394
 395         if (req->cryptlen <= AES_BLOCK_SIZE) {
 396                 if (req->cryptlen < AES_BLOCK_SIZE)
 397                         return -EINVAL;
 398                 cbc_blocks = 1;
 399         }
 400
 401         if (cbc_blocks > 0) {
 402                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 403                                            cbc_blocks * AES_BLOCK_SIZE,
 404                                            req->iv);
 405
 406                 err = cbc_encrypt(&subreq);
 407                 if (err)
 408                         return err;
 409
 410                 if (req->cryptlen == AES_BLOCK_SIZE)
 411                         return 0;
 412
 413                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 414                 if (req->dst != req->src)
 415                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 416                                                subreq.cryptlen);
 417         }
 418
 419         /* handle ciphertext stealing */
 420         skcipher_request_set_crypt(&subreq, src, dst,
 421                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 422                                    req->iv);
 423
 424         err = skcipher_walk_virt(&walk, &subreq, false);
 425         if (err)
 426                 return err;
 427
 428         kernel_fpu_begin();
 429         aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 430                           walk.nbytes, walk.iv);
 431         kernel_fpu_end();
 432
 433         return skcipher_walk_done(&walk, 0);
 434 }
 435
 436 static int cts_cbc_decrypt(struct skcipher_request *req)
 437 {
 438         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 439         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 440         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 441         struct scatterlist *src = req->src, *dst = req->dst;
 442         struct scatterlist sg_src[2], sg_dst[2];
 443         struct skcipher_request subreq;
 444         struct skcipher_walk walk;
 445         int err;
 446
 447         skcipher_request_set_tfm(&subreq, tfm);
 448         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 449                                       NULL, NULL);
 450
 451         if (req->cryptlen <= AES_BLOCK_SIZE) {
 452                 if (req->cryptlen < AES_BLOCK_SIZE)
 453                         return -EINVAL;
 454                 cbc_blocks = 1;
 455         }
 456
 457         if (cbc_blocks > 0) {
 458                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 459                                            cbc_blocks * AES_BLOCK_SIZE,
 460                                            req->iv);
 461
 462                 err = cbc_decrypt(&subreq);
 463                 if (err)
 464                         return err;
 465
 466                 if (req->cryptlen == AES_BLOCK_SIZE)
 467                         return 0;
 468
 469                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 470                 if (req->dst != req->src)
 471                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 472                                                subreq.cryptlen);
 473         }
 474
 475         /* handle ciphertext stealing */
 476         skcipher_request_set_crypt(&subreq, src, dst,
 477                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 478                                    req->iv);
 479
 480         err = skcipher_walk_virt(&walk, &subreq, false);
 481         if (err)
 482                 return err;
 483
 484         kernel_fpu_begin();
 485         aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 486                           walk.nbytes, walk.iv);
 487         kernel_fpu_end();
 488
 489         return skcipher_walk_done(&walk, 0);
 490 }
 491
 492 #ifdef CONFIG_X86_64
 493 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 494                               const u8 *in, unsigned int len, u8 *iv)
 495 {
 496         /*
 497          * based on key length, override with the by8 version
 498          * of ctr mode encryption/decryption for improved performance
 499          * aes_set_key_common() ensures that key length is one of
 500          * {128,192,256}
 501          */
 502         if (ctx->key_length == AES_KEYSIZE_128)
 503                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 504         else if (ctx->key_length == AES_KEYSIZE_192)
 505                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 506         else
 507                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 508 }
 509
 510 static int ctr_crypt(struct skcipher_request *req)
 511 {
 512         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 513         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 514         u8 keystream[AES_BLOCK_SIZE];
 515         struct skcipher_walk walk;
 516         unsigned int nbytes;
 517         int err;
 518
 519         err = skcipher_walk_virt(&walk, req, false);
 520
 521         while ((nbytes = walk.nbytes) > 0) {
 522                 kernel_fpu_begin();
 523                 if (nbytes & AES_BLOCK_MASK)
 524                         static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
 525                                                        walk.src.virt.addr,
 526                                                        nbytes & AES_BLOCK_MASK,
 527                                                        walk.iv);
 528                 nbytes &= ~AES_BLOCK_MASK;
 529
 530                 if (walk.nbytes == walk.total && nbytes > 0) {
 531                         aesni_enc(ctx, keystream, walk.iv);
 532                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
 533                                        walk.src.virt.addr + walk.nbytes - nbytes,
 534                                        keystream, nbytes);
 535                         crypto_inc(walk.iv, AES_BLOCK_SIZE);
 536                         nbytes = 0;
 537                 }
 538                 kernel_fpu_end();
 539                 err = skcipher_walk_done(&walk, nbytes);
 540         }
 541         return err;
 542 }
 543
 544 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 545                                    const u8 *in, unsigned int len, u8 *iv,
 546                                    unsigned int byte_ctr)
 547 {
 548         if (ctx->key_length == AES_KEYSIZE_128)
 549                 aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
 550                                          byte_ctr);
 551         else if (ctx->key_length == AES_KEYSIZE_192)
 552                 aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
 553                                          byte_ctr);
 554         else
 555                 aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
 556                                          byte_ctr);
 557 }
 558
 559 static int xctr_crypt(struct skcipher_request *req)
 560 {
 561         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 562         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 563         u8 keystream[AES_BLOCK_SIZE];
 564         struct skcipher_walk walk;
 565         unsigned int nbytes;
 566         unsigned int byte_ctr = 0;
 567         int err;
 568         __le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
 569
 570         err = skcipher_walk_virt(&walk, req, false);
 571
 572         while ((nbytes = walk.nbytes) > 0) {
 573                 kernel_fpu_begin();
 574                 if (nbytes & AES_BLOCK_MASK)
 575                         aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
 576                                 walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
 577                                 walk.iv, byte_ctr);
 578                 nbytes &= ~AES_BLOCK_MASK;
 579                 byte_ctr += walk.nbytes - nbytes;
 580
 581                 if (walk.nbytes == walk.total && nbytes > 0) {
 582                         memcpy(block, walk.iv, AES_BLOCK_SIZE);
 583                         block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
 584                         aesni_enc(ctx, keystream, (u8 *)block);
 585                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
 586                                        nbytes, walk.src.virt.addr + walk.nbytes
 587                                        - nbytes, keystream, nbytes);
 588                         byte_ctr += nbytes;
 589                         nbytes = 0;
 590                 }
 591                 kernel_fpu_end();
 592                 err = skcipher_walk_done(&walk, nbytes);
 593         }
 594         return err;
 595 }
 596
 597 static int
 598 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 599 {
 600         struct crypto_aes_ctx ctx;
 601         int ret;
 602
 603         ret = aes_expandkey(&ctx, key, key_len);
 604         if (ret)
 605                 return ret;
 606
 607         /* Clear the data in the hash sub key container to zero.*/
 608         /* We want to cipher all zeros to create the hash sub key. */
 609         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 610
 611         aes_encrypt(&ctx, hash_subkey, hash_subkey);
 612
 613         memzero_explicit(&ctx, sizeof(ctx));
 614         return 0;
 615 }
 616
 617 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 618                                   unsigned int key_len)
 619 {
 620         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 621
 622         if (key_len < 4)
 623                 return -EINVAL;
 624
 625         /*Account for 4 byte nonce at the end.*/
 626         key_len -= 4;
 627
 628         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 629
 630         return aes_set_key_common(crypto_aead_tfm(aead),
 631                                   &ctx->aes_key_expanded, key, key_len) ?:
 632                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 633 }
 634
 635 /* This is the Integrity Check Value (aka the authentication tag) length and can
 636  * be 8, 12 or 16 bytes long. */
 637 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 638                                        unsigned int authsize)
 639 {
 640         switch (authsize) {
 641         case 8:
 642         case 12:
 643         case 16:
 644                 break;
 645         default:
 646                 return -EINVAL;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 653                                        unsigned int authsize)
 654 {
 655         switch (authsize) {
 656         case 4:
 657         case 8:
 658         case 12:
 659         case 13:
 660         case 14:
 661         case 15:
 662         case 16:
 663                 break;
 664         default:
 665                 return -EINVAL;
 666         }
 667
 668         return 0;
 669 }
 670
 671 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 672                               unsigned int assoclen, u8 *hash_subkey,
 673                               u8 *iv, void *aes_ctx, u8 *auth_tag,
 674                               unsigned long auth_tag_len)
 675 {
 676         u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
 677         struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
 678         unsigned long left = req->cryptlen;
 679         struct scatter_walk assoc_sg_walk;
 680         struct skcipher_walk walk;
 681         bool do_avx, do_avx2;
 682         u8 *assocmem = NULL;
 683         u8 *assoc;
 684         int err;
 685
 686         if (!enc)
 687                 left -= auth_tag_len;
 688
 689         do_avx = (left >= AVX_GEN2_OPTSIZE);
 690         do_avx2 = (left >= AVX_GEN4_OPTSIZE);
 691
 692         /* Linearize assoc, if not already linear */
 693         if (req->src->length >= assoclen && req->src->length) {
 694                 scatterwalk_start(&assoc_sg_walk, req->src);
 695                 assoc = scatterwalk_map(&assoc_sg_walk);
 696         } else {
 697                 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 698                               GFP_KERNEL : GFP_ATOMIC;
 699
 700                 /* assoc can be any length, so must be on heap */
 701                 assocmem = kmalloc(assoclen, flags);
 702                 if (unlikely(!assocmem))
 703                         return -ENOMEM;
 704                 assoc = assocmem;
 705
 706                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 707         }
 708
 709         kernel_fpu_begin();
 710         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 711                 aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc,
 712                                         assoclen);
 713         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 714                 aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc,
 715                                         assoclen);
 716         else
 717                 aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
 718         kernel_fpu_end();
 719
 720         if (!assocmem)
 721                 scatterwalk_unmap(assoc);
 722         else
 723                 kfree(assocmem);
 724
 725         err = enc ? skcipher_walk_aead_encrypt(&walk, req, false)
 726                   : skcipher_walk_aead_decrypt(&walk, req, false);
 727
 728         while (walk.nbytes > 0) {
 729                 kernel_fpu_begin();
 730                 if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
 731                         if (enc)
 732                                 aesni_gcm_enc_update_avx_gen4(aes_ctx, data,
 733                                                               walk.dst.virt.addr,
 734                                                               walk.src.virt.addr,
 735                                                               walk.nbytes);
 736                         else
 737                                 aesni_gcm_dec_update_avx_gen4(aes_ctx, data,
 738                                                               walk.dst.virt.addr,
 739                                                               walk.src.virt.addr,
 740                                                               walk.nbytes);
 741                 } else if (static_branch_likely(&gcm_use_avx) && do_avx) {
 742                         if (enc)
 743                                 aesni_gcm_enc_update_avx_gen2(aes_ctx, data,
 744                                                               walk.dst.virt.addr,
 745                                                               walk.src.virt.addr,
 746                                                               walk.nbytes);
 747                         else
 748                                 aesni_gcm_dec_update_avx_gen2(aes_ctx, data,
 749                                                               walk.dst.virt.addr,
 750                                                               walk.src.virt.addr,
 751                                                               walk.nbytes);
 752                 } else if (enc) {
 753                         aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr,
 754                                              walk.src.virt.addr, walk.nbytes);
 755                 } else {
 756                         aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr,
 757                                              walk.src.virt.addr, walk.nbytes);
 758                 }
 759                 kernel_fpu_end();
 760
 761                 err = skcipher_walk_done(&walk, 0);
 762         }
 763
 764         if (err)
 765                 return err;
 766
 767         kernel_fpu_begin();
 768         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 769                 aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag,
 770                                             auth_tag_len);
 771         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 772                 aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag,
 773                                             auth_tag_len);
 774         else
 775                 aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len);
 776         kernel_fpu_end();
 777
 778         return 0;
 779 }
 780
 781 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 782                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 783 {
 784         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 785         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 786         u8 auth_tag[16];
 787         int err;
 788
 789         err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx,
 790                                  auth_tag, auth_tag_len);
 791         if (err)
 792                 return err;
 793
 794         scatterwalk_map_and_copy(auth_tag, req->dst,
 795                                  req->assoclen + req->cryptlen,
 796                                  auth_tag_len, 1);
 797         return 0;
 798 }
 799
 800 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 801                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 802 {
 803         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 804         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 805         u8 auth_tag_msg[16];
 806         u8 auth_tag[16];
 807         int err;
 808
 809         err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx,
 810                                  auth_tag, auth_tag_len);
 811         if (err)
 812                 return err;
 813
 814         /* Copy out original auth_tag */
 815         scatterwalk_map_and_copy(auth_tag_msg, req->src,
 816                                  req->assoclen + req->cryptlen - auth_tag_len,
 817                                  auth_tag_len, 0);
 818
 819         /* Compare generated tag with passed in tag. */
 820         if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) {
 821                 memzero_explicit(auth_tag, sizeof(auth_tag));
 822                 return -EBADMSG;
 823         }
 824         return 0;
 825 }
 826
 827 static int helper_rfc4106_encrypt(struct aead_request *req)
 828 {
 829         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 830         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 831         void *aes_ctx = &(ctx->aes_key_expanded);
 832         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 833         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 834         unsigned int i;
 835         __be32 counter = cpu_to_be32(1);
 836
 837         /* Assuming we are supporting rfc4106 64-bit extended */
 838         /* sequence numbers We need to have the AAD length equal */
 839         /* to 16 or 20 bytes */
 840         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 841                 return -EINVAL;
 842
 843         /* IV below built */
 844         for (i = 0; i < 4; i++)
 845                 *(iv+i) = ctx->nonce[i];
 846         for (i = 0; i < 8; i++)
 847                 *(iv+4+i) = req->iv[i];
 848         *((__be32 *)(iv+12)) = counter;
 849
 850         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 851                               aes_ctx);
 852 }
 853
 854 static int helper_rfc4106_decrypt(struct aead_request *req)
 855 {
 856         __be32 counter = cpu_to_be32(1);
 857         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 858         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 859         void *aes_ctx = &(ctx->aes_key_expanded);
 860         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 861         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 862         unsigned int i;
 863
 864         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 865                 return -EINVAL;
 866
 867         /* Assuming we are supporting rfc4106 64-bit extended */
 868         /* sequence numbers We need to have the AAD length */
 869         /* equal to 16 or 20 bytes */
 870
 871         /* IV below built */
 872         for (i = 0; i < 4; i++)
 873                 *(iv+i) = ctx->nonce[i];
 874         for (i = 0; i < 8; i++)
 875                 *(iv+4+i) = req->iv[i];
 876         *((__be32 *)(iv+12)) = counter;
 877
 878         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 879                               aes_ctx);
 880 }
 881 #endif
 882
 883 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 884                             unsigned int keylen)
 885 {
 886         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 887         int err;
 888
 889         err = xts_verify_key(tfm, key, keylen);
 890         if (err)
 891                 return err;
 892
 893         keylen /= 2;
 894
 895         /* first half of xts-key is for crypt */
 896         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 897                                  key, keylen);
 898         if (err)
 899                 return err;
 900
 901         /* second half of xts-key is for tweak */
 902         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 903                                   key + keylen, keylen);
 904 }
 905
 906 static int xts_crypt(struct skcipher_request *req, bool encrypt)
 907 {
 908         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 909         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 910         int tail = req->cryptlen % AES_BLOCK_SIZE;
 911         struct skcipher_request subreq;
 912         struct skcipher_walk walk;
 913         int err;
 914
 915         if (req->cryptlen < AES_BLOCK_SIZE)
 916                 return -EINVAL;
 917
 918         err = skcipher_walk_virt(&walk, req, false);
 919         if (!walk.nbytes)
 920                 return err;
 921
 922         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 923                 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 924
 925                 skcipher_walk_abort(&walk);
 926
 927                 skcipher_request_set_tfm(&subreq, tfm);
 928                 skcipher_request_set_callback(&subreq,
 929                                               skcipher_request_flags(req),
 930                                               NULL, NULL);
 931                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 932                                            blocks * AES_BLOCK_SIZE, req->iv);
 933                 req = &subreq;
 934
 935                 err = skcipher_walk_virt(&walk, req, false);
 936                 if (!walk.nbytes)
 937                         return err;
 938         } else {
 939                 tail = 0;
 940         }
 941
 942         kernel_fpu_begin();
 943
 944         /* calculate first value of T */
 945         aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv);
 946
 947         while (walk.nbytes > 0) {
 948                 int nbytes = walk.nbytes;
 949
 950                 if (nbytes < walk.total)
 951                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 952
 953                 if (encrypt)
 954                         aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
 955                                           walk.dst.virt.addr, walk.src.virt.addr,
 956                                           nbytes, walk.iv);
 957                 else
 958                         aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
 959                                           walk.dst.virt.addr, walk.src.virt.addr,
 960                                           nbytes, walk.iv);
 961                 kernel_fpu_end();
 962
 963                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 964
 965                 if (walk.nbytes > 0)
 966                         kernel_fpu_begin();
 967         }
 968
 969         if (unlikely(tail > 0 && !err)) {
 970                 struct scatterlist sg_src[2], sg_dst[2];
 971                 struct scatterlist *src, *dst;
 972
 973                 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 974                 if (req->dst != req->src)
 975                         dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 976
 977                 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 978                                            req->iv);
 979
 980                 err = skcipher_walk_virt(&walk, &subreq, false);
 981                 if (err)
 982                         return err;
 983
 984                 kernel_fpu_begin();
 985                 if (encrypt)
 986                         aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
 987                                           walk.dst.virt.addr, walk.src.virt.addr,
 988                                           walk.nbytes, walk.iv);
 989                 else
 990                         aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
 991                                           walk.dst.virt.addr, walk.src.virt.addr,
 992                                           walk.nbytes, walk.iv);
 993                 kernel_fpu_end();
 994
 995                 err = skcipher_walk_done(&walk, 0);
 996         }
 997         return err;
 998 }
 999
1000 static int xts_encrypt(struct skcipher_request *req)
1001 {
1002         return xts_crypt(req, true);
1003 }
1004
1005 static int xts_decrypt(struct skcipher_request *req)
1006 {
1007         return xts_crypt(req, false);
1008 }
1009
1010 static struct crypto_alg aesni_cipher_alg = {
1011         .cra_name               = "aes",
1012         .cra_driver_name        = "aes-aesni",
1013         .cra_priority           = 300,
1014         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1015         .cra_blocksize          = AES_BLOCK_SIZE,
1016         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1017         .cra_module             = THIS_MODULE,
1018         .cra_u  = {
1019                 .cipher = {
1020                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1021                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1022                         .cia_setkey             = aes_set_key,
1023                         .cia_encrypt            = aesni_encrypt,
1024                         .cia_decrypt            = aesni_decrypt
1025                 }
1026         }
1027 };
1028
1029 static struct skcipher_alg aesni_skciphers[] = {
1030         {
1031                 .base = {
1032                         .cra_name               = "__ecb(aes)",
1033                         .cra_driver_name        = "__ecb-aes-aesni",
1034                         .cra_priority           = 400,
1035                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1036                         .cra_blocksize          = AES_BLOCK_SIZE,
1037                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1038                         .cra_module             = THIS_MODULE,
1039                 },
1040                 .min_keysize    = AES_MIN_KEY_SIZE,
1041                 .max_keysize    = AES_MAX_KEY_SIZE,
1042                 .setkey         = aesni_skcipher_setkey,
1043                 .encrypt        = ecb_encrypt,
1044                 .decrypt        = ecb_decrypt,
1045         }, {
1046                 .base = {
1047                         .cra_name               = "__cbc(aes)",
1048                         .cra_driver_name        = "__cbc-aes-aesni",
1049                         .cra_priority           = 400,
1050                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1051                         .cra_blocksize          = AES_BLOCK_SIZE,
1052                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1053                         .cra_module             = THIS_MODULE,
1054                 },
1055                 .min_keysize    = AES_MIN_KEY_SIZE,
1056                 .max_keysize    = AES_MAX_KEY_SIZE,
1057                 .ivsize         = AES_BLOCK_SIZE,
1058                 .setkey         = aesni_skcipher_setkey,
1059                 .encrypt        = cbc_encrypt,
1060                 .decrypt        = cbc_decrypt,
1061         }, {
1062                 .base = {
1063                         .cra_name               = "__cts(cbc(aes))",
1064                         .cra_driver_name        = "__cts-cbc-aes-aesni",
1065                         .cra_priority           = 400,
1066                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1067                         .cra_blocksize          = AES_BLOCK_SIZE,
1068                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1069                         .cra_module             = THIS_MODULE,
1070                 },
1071                 .min_keysize    = AES_MIN_KEY_SIZE,
1072                 .max_keysize    = AES_MAX_KEY_SIZE,
1073                 .ivsize         = AES_BLOCK_SIZE,
1074                 .walksize       = 2 * AES_BLOCK_SIZE,
1075                 .setkey         = aesni_skcipher_setkey,
1076                 .encrypt        = cts_cbc_encrypt,
1077                 .decrypt        = cts_cbc_decrypt,
1078 #ifdef CONFIG_X86_64
1079         }, {
1080                 .base = {
1081                         .cra_name               = "__ctr(aes)",
1082                         .cra_driver_name        = "__ctr-aes-aesni",
1083                         .cra_priority           = 400,
1084                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1085                         .cra_blocksize          = 1,
1086                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1087                         .cra_module             = THIS_MODULE,
1088                 },
1089                 .min_keysize    = AES_MIN_KEY_SIZE,
1090                 .max_keysize    = AES_MAX_KEY_SIZE,
1091                 .ivsize         = AES_BLOCK_SIZE,
1092                 .chunksize      = AES_BLOCK_SIZE,
1093                 .setkey         = aesni_skcipher_setkey,
1094                 .encrypt        = ctr_crypt,
1095                 .decrypt        = ctr_crypt,
1096 #endif
1097         }, {
1098                 .base = {
1099                         .cra_name               = "__xts(aes)",
1100                         .cra_driver_name        = "__xts-aes-aesni",
1101                         .cra_priority           = 401,
1102                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1103                         .cra_blocksize          = AES_BLOCK_SIZE,
1104                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1105                         .cra_module             = THIS_MODULE,
1106                 },
1107                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1108                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1109                 .ivsize         = AES_BLOCK_SIZE,
1110                 .walksize       = 2 * AES_BLOCK_SIZE,
1111                 .setkey         = xts_aesni_setkey,
1112                 .encrypt        = xts_encrypt,
1113                 .decrypt        = xts_decrypt,
1114         }
1115 };
1116
1117 static
1118 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1119
1120 #ifdef CONFIG_X86_64
1121 /*
1122  * XCTR does not have a non-AVX implementation, so it must be enabled
1123  * conditionally.
1124  */
1125 static struct skcipher_alg aesni_xctr = {
1126         .base = {
1127                 .cra_name               = "__xctr(aes)",
1128                 .cra_driver_name        = "__xctr-aes-aesni",
1129                 .cra_priority           = 400,
1130                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1131                 .cra_blocksize          = 1,
1132                 .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1133                 .cra_module             = THIS_MODULE,
1134         },
1135         .min_keysize    = AES_MIN_KEY_SIZE,
1136         .max_keysize    = AES_MAX_KEY_SIZE,
1137         .ivsize         = AES_BLOCK_SIZE,
1138         .chunksize      = AES_BLOCK_SIZE,
1139         .setkey         = aesni_skcipher_setkey,
1140         .encrypt        = xctr_crypt,
1141         .decrypt        = xctr_crypt,
1142 };
1143
1144 static struct simd_skcipher_alg *aesni_simd_xctr;
1145 #endif /* CONFIG_X86_64 */
1146
1147 #ifdef CONFIG_X86_64
1148 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1149                                   unsigned int key_len)
1150 {
1151         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1152
1153         return aes_set_key_common(crypto_aead_tfm(aead),
1154                                   &ctx->aes_key_expanded, key, key_len) ?:
1155                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1156 }
1157
1158 static int generic_gcmaes_encrypt(struct aead_request *req)
1159 {
1160         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1161         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1162         void *aes_ctx = &(ctx->aes_key_expanded);
1163         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1164         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1165         __be32 counter = cpu_to_be32(1);
1166
1167         memcpy(iv, req->iv, 12);
1168         *((__be32 *)(iv+12)) = counter;
1169
1170         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1171                               aes_ctx);
1172 }
1173
1174 static int generic_gcmaes_decrypt(struct aead_request *req)
1175 {
1176         __be32 counter = cpu_to_be32(1);
1177         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1178         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1179         void *aes_ctx = &(ctx->aes_key_expanded);
1180         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1181         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1182
1183         memcpy(iv, req->iv, 12);
1184         *((__be32 *)(iv+12)) = counter;
1185
1186         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1187                               aes_ctx);
1188 }
1189
1190 static struct aead_alg aesni_aeads[] = { {
1191         .setkey                 = common_rfc4106_set_key,
1192         .setauthsize            = common_rfc4106_set_authsize,
1193         .encrypt                = helper_rfc4106_encrypt,
1194         .decrypt                = helper_rfc4106_decrypt,
1195         .ivsize                 = GCM_RFC4106_IV_SIZE,
1196         .maxauthsize            = 16,
1197         .base = {
1198                 .cra_name               = "__rfc4106(gcm(aes))",
1199                 .cra_driver_name        = "__rfc4106-gcm-aesni",
1200                 .cra_priority           = 400,
1201                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1202                 .cra_blocksize          = 1,
1203                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1204                 .cra_alignmask          = 0,
1205                 .cra_module             = THIS_MODULE,
1206         },
1207 }, {
1208         .setkey                 = generic_gcmaes_set_key,
1209         .setauthsize            = generic_gcmaes_set_authsize,
1210         .encrypt                = generic_gcmaes_encrypt,
1211         .decrypt                = generic_gcmaes_decrypt,
1212         .ivsize                 = GCM_AES_IV_SIZE,
1213         .maxauthsize            = 16,
1214         .base = {
1215                 .cra_name               = "__gcm(aes)",
1216                 .cra_driver_name        = "__generic-gcm-aesni",
1217                 .cra_priority           = 400,
1218                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1219                 .cra_blocksize          = 1,
1220                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1221                 .cra_alignmask          = 0,
1222                 .cra_module             = THIS_MODULE,
1223         },
1224 } };
1225 #else
1226 static struct aead_alg aesni_aeads[0];
1227 #endif
1228
1229 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1230
1231 static const struct x86_cpu_id aesni_cpu_id[] = {
1232         X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1233         {}
1234 };
1235 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1236
1237 static int __init aesni_init(void)
1238 {
1239         int err;
1240
1241         if (!x86_match_cpu(aesni_cpu_id))
1242                 return -ENODEV;
1243 #ifdef CONFIG_X86_64
1244         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1245                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1246                 static_branch_enable(&gcm_use_avx);
1247                 static_branch_enable(&gcm_use_avx2);
1248         } else
1249         if (boot_cpu_has(X86_FEATURE_AVX)) {
1250                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1251                 static_branch_enable(&gcm_use_avx);
1252         } else {
1253                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1254         }
1255         if (boot_cpu_has(X86_FEATURE_AVX)) {
1256                 /* optimize performance of ctr mode encryption transform */
1257                 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1258                 pr_info("AES CTR mode by8 optimization enabled\n");
1259         }
1260 #endif /* CONFIG_X86_64 */
1261
1262         err = crypto_register_alg(&aesni_cipher_alg);
1263         if (err)
1264                 return err;
1265
1266         err = simd_register_skciphers_compat(aesni_skciphers,
1267                                              ARRAY_SIZE(aesni_skciphers),
1268                                              aesni_simd_skciphers);
1269         if (err)
1270                 goto unregister_cipher;
1271
1272         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1273                                          aesni_simd_aeads);
1274         if (err)
1275                 goto unregister_skciphers;
1276
1277 #ifdef CONFIG_X86_64
1278         if (boot_cpu_has(X86_FEATURE_AVX))
1279                 err = simd_register_skciphers_compat(&aesni_xctr, 1,
1280                                                      &aesni_simd_xctr);
1281         if (err)
1282                 goto unregister_aeads;
1283 #endif /* CONFIG_X86_64 */
1284
1285         return 0;
1286
1287 #ifdef CONFIG_X86_64
1288 unregister_aeads:
1289         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1290                                 aesni_simd_aeads);
1291 #endif /* CONFIG_X86_64 */
1292
1293 unregister_skciphers:
1294         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1295                                   aesni_simd_skciphers);
1296 unregister_cipher:
1297         crypto_unregister_alg(&aesni_cipher_alg);
1298         return err;
1299 }
1300
1301 static void __exit aesni_exit(void)
1302 {
1303         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1304                               aesni_simd_aeads);
1305         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1306                                   aesni_simd_skciphers);
1307         crypto_unregister_alg(&aesni_cipher_alg);
1308 #ifdef CONFIG_X86_64
1309         if (boot_cpu_has(X86_FEATURE_AVX))
1310                 simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
1311 #endif /* CONFIG_X86_64 */
1312 }
1313
1314 late_initcall(aesni_init);
1315 module_exit(aesni_exit);
1316
1317 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1318 MODULE_LICENSE("GPL");
1319 MODULE_ALIAS_CRYPTO("aes");