drivers/crypto/caam/caamalg.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * caam - Freescale FSL CAAM support for crypto API
   4  *
   5  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   6  * Copyright 2016-2019 NXP
   7  *
   8  * Based on talitos crypto API driver.
   9  *
  10  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
  11  *
  12  * ---------------                     ---------------
  13  * | JobDesc #1  |-------------------->|  ShareDesc  |
  14  * | *(packet 1) |                     |   (PDB)     |
  15  * ---------------      |------------->|  (hashKey)  |
  16  *       .              |              | (cipherKey) |
  17  *       .              |    |-------->| (operation) |
  18  * ---------------      |    |         ---------------
  19  * | JobDesc #2  |------|    |
  20  * | *(packet 2) |           |
  21  * ---------------           |
  22  *       .                   |
  23  *       .                   |
  24  * ---------------           |
  25  * | JobDesc #3  |------------
  26  * | *(packet 3) |
  27  * ---------------
  28  *
  29  * The SharedDesc never changes for a connection unless rekeyed, but
  30  * each packet will likely be in a different place. So all we need
  31  * to know to process the packet is where the input is, where the
  32  * output goes, and what context we want to process with. Context is
  33  * in the SharedDesc, packet references in the JobDesc.
  34  *
  35  * So, a job desc looks like:
  36  *
  37  * ---------------------
  38  * | Header            |
  39  * | ShareDesc Pointer |
  40  * | SEQ_OUT_PTR       |
  41  * | (output buffer)   |
  42  * | (output length)   |
  43  * | SEQ_IN_PTR        |
  44  * | (input buffer)    |
  45  * | (input length)    |
  46  * ---------------------
  47  */
  48
  49 #include "compat.h"
  50
  51 #include "regs.h"
  52 #include "intern.h"
  53 #include "desc_constr.h"
  54 #include "jr.h"
  55 #include "error.h"
  56 #include "sg_sw_sec4.h"
  57 #include "key_gen.h"
  58 #include "caamalg_desc.h"
  59 #include <crypto/engine.h>
  60 #include <crypto/xts.h>
  61 #include <asm/unaligned.h>
  62
  63 /*
  64  * crypto alg
  65  */
  66 #define CAAM_CRA_PRIORITY               3000
  67 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  68 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  69                                          CTR_RFC3686_NONCE_SIZE + \
  70                                          SHA512_DIGEST_SIZE * 2)
  71
  72 #define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  73 #define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  74                                          CAAM_CMD_SZ * 4)
  75 #define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  76                                          CAAM_CMD_SZ * 5)
  77
  78 #define CHACHAPOLY_DESC_JOB_IO_LEN      (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
  79
  80 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
  81 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  82
  83 struct caam_alg_entry {
  84         int class1_alg_type;
  85         int class2_alg_type;
  86         bool rfc3686;
  87         bool geniv;
  88         bool nodkp;
  89 };
  90
  91 struct caam_aead_alg {
  92         struct aead_alg aead;
  93         struct caam_alg_entry caam;
  94         bool registered;
  95 };
  96
  97 struct caam_skcipher_alg {
  98         struct skcipher_alg skcipher;
  99         struct caam_alg_entry caam;
 100         bool registered;
 101 };
 102
 103 /*
 104  * per-session context
 105  */
 106 struct caam_ctx {
 107         struct crypto_engine_ctx enginectx;
 108         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 109         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 110         u8 key[CAAM_MAX_KEY_SIZE];
 111         dma_addr_t sh_desc_enc_dma;
 112         dma_addr_t sh_desc_dec_dma;
 113         dma_addr_t key_dma;
 114         enum dma_data_direction dir;
 115         struct device *jrdev;
 116         struct alginfo adata;
 117         struct alginfo cdata;
 118         unsigned int authsize;
 119         bool xts_key_fallback;
 120         struct crypto_skcipher *fallback;
 121 };
 122
 123 struct caam_skcipher_req_ctx {
 124         struct skcipher_edesc *edesc;
 125         struct skcipher_request fallback_req;
 126 };
 127
 128 struct caam_aead_req_ctx {
 129         struct aead_edesc *edesc;
 130 };
 131
 132 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 133 {
 134         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 135         struct device *jrdev = ctx->jrdev;
 136         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 137         u32 *desc;
 138         int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
 139                         ctx->adata.keylen_pad;
 140
 141         /*
 142          * Job Descriptor and Shared Descriptors
 143          * must all fit into the 64-word Descriptor h/w Buffer
 144          */
 145         if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
 146                 ctx->adata.key_inline = true;
 147                 ctx->adata.key_virt = ctx->key;
 148         } else {
 149                 ctx->adata.key_inline = false;
 150                 ctx->adata.key_dma = ctx->key_dma;
 151         }
 152
 153         /* aead_encrypt shared descriptor */
 154         desc = ctx->sh_desc_enc;
 155         cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
 156                                     ctrlpriv->era);
 157         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 158                                    desc_bytes(desc), ctx->dir);
 159
 160         /*
 161          * Job Descriptor and Shared Descriptors
 162          * must all fit into the 64-word Descriptor h/w Buffer
 163          */
 164         if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
 165                 ctx->adata.key_inline = true;
 166                 ctx->adata.key_virt = ctx->key;
 167         } else {
 168                 ctx->adata.key_inline = false;
 169                 ctx->adata.key_dma = ctx->key_dma;
 170         }
 171
 172         /* aead_decrypt shared descriptor */
 173         desc = ctx->sh_desc_dec;
 174         cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
 175                                     ctrlpriv->era);
 176         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 177                                    desc_bytes(desc), ctx->dir);
 178
 179         return 0;
 180 }
 181
 182 static int aead_set_sh_desc(struct crypto_aead *aead)
 183 {
 184         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 185                                                  struct caam_aead_alg, aead);
 186         unsigned int ivsize = crypto_aead_ivsize(aead);
 187         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 188         struct device *jrdev = ctx->jrdev;
 189         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 190         u32 ctx1_iv_off = 0;
 191         u32 *desc, *nonce = NULL;
 192         u32 inl_mask;
 193         unsigned int data_len[2];
 194         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 195                                OP_ALG_AAI_CTR_MOD128);
 196         const bool is_rfc3686 = alg->caam.rfc3686;
 197
 198         if (!ctx->authsize)
 199                 return 0;
 200
 201         /* NULL encryption / decryption */
 202         if (!ctx->cdata.keylen)
 203                 return aead_null_set_sh_desc(aead);
 204
 205         /*
 206          * AES-CTR needs to load IV in CONTEXT1 reg
 207          * at an offset of 128bits (16bytes)
 208          * CONTEXT1[255:128] = IV
 209          */
 210         if (ctr_mode)
 211                 ctx1_iv_off = 16;
 212
 213         /*
 214          * RFC3686 specific:
 215          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 216          */
 217         if (is_rfc3686) {
 218                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 219                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 220                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 221         }
 222
 223         /*
 224          * In case |user key| > |derived key|, using DKP<imm,imm>
 225          * would result in invalid opcodes (last bytes of user key) in
 226          * the resulting descriptor. Use DKP<ptr,imm> instead => both
 227          * virtual and dma key addresses are needed.
 228          */
 229         ctx->adata.key_virt = ctx->key;
 230         ctx->adata.key_dma = ctx->key_dma;
 231
 232         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 233         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 234
 235         data_len[0] = ctx->adata.keylen_pad;
 236         data_len[1] = ctx->cdata.keylen;
 237
 238         if (alg->caam.geniv)
 239                 goto skip_enc;
 240
 241         /*
 242          * Job Descriptor and Shared Descriptors
 243          * must all fit into the 64-word Descriptor h/w Buffer
 244          */
 245         if (desc_inline_query(DESC_AEAD_ENC_LEN +
 246                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 247                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 248                               ARRAY_SIZE(data_len)) < 0)
 249                 return -EINVAL;
 250
 251         ctx->adata.key_inline = !!(inl_mask & 1);
 252         ctx->cdata.key_inline = !!(inl_mask & 2);
 253
 254         /* aead_encrypt shared descriptor */
 255         desc = ctx->sh_desc_enc;
 256         cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
 257                                ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
 258                                false, ctrlpriv->era);
 259         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 260                                    desc_bytes(desc), ctx->dir);
 261
 262 skip_enc:
 263         /*
 264          * Job Descriptor and Shared Descriptors
 265          * must all fit into the 64-word Descriptor h/w Buffer
 266          */
 267         if (desc_inline_query(DESC_AEAD_DEC_LEN +
 268                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 269                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 270                               ARRAY_SIZE(data_len)) < 0)
 271                 return -EINVAL;
 272
 273         ctx->adata.key_inline = !!(inl_mask & 1);
 274         ctx->cdata.key_inline = !!(inl_mask & 2);
 275
 276         /* aead_decrypt shared descriptor */
 277         desc = ctx->sh_desc_dec;
 278         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
 279                                ctx->authsize, alg->caam.geniv, is_rfc3686,
 280                                nonce, ctx1_iv_off, false, ctrlpriv->era);
 281         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 282                                    desc_bytes(desc), ctx->dir);
 283
 284         if (!alg->caam.geniv)
 285                 goto skip_givenc;
 286
 287         /*
 288          * Job Descriptor and Shared Descriptors
 289          * must all fit into the 64-word Descriptor h/w Buffer
 290          */
 291         if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
 292                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 293                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 294                               ARRAY_SIZE(data_len)) < 0)
 295                 return -EINVAL;
 296
 297         ctx->adata.key_inline = !!(inl_mask & 1);
 298         ctx->cdata.key_inline = !!(inl_mask & 2);
 299
 300         /* aead_givencrypt shared descriptor */
 301         desc = ctx->sh_desc_enc;
 302         cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
 303                                   ctx->authsize, is_rfc3686, nonce,
 304                                   ctx1_iv_off, false, ctrlpriv->era);
 305         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 306                                    desc_bytes(desc), ctx->dir);
 307
 308 skip_givenc:
 309         return 0;
 310 }
 311
 312 static int aead_setauthsize(struct crypto_aead *authenc,
 313                                     unsigned int authsize)
 314 {
 315         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 316
 317         ctx->authsize = authsize;
 318         aead_set_sh_desc(authenc);
 319
 320         return 0;
 321 }
 322
 323 static int gcm_set_sh_desc(struct crypto_aead *aead)
 324 {
 325         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 326         struct device *jrdev = ctx->jrdev;
 327         unsigned int ivsize = crypto_aead_ivsize(aead);
 328         u32 *desc;
 329         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 330                         ctx->cdata.keylen;
 331
 332         if (!ctx->cdata.keylen || !ctx->authsize)
 333                 return 0;
 334
 335         /*
 336          * AES GCM encrypt shared descriptor
 337          * Job Descriptor and Shared Descriptor
 338          * must fit into the 64-word Descriptor h/w Buffer
 339          */
 340         if (rem_bytes >= DESC_GCM_ENC_LEN) {
 341                 ctx->cdata.key_inline = true;
 342                 ctx->cdata.key_virt = ctx->key;
 343         } else {
 344                 ctx->cdata.key_inline = false;
 345                 ctx->cdata.key_dma = ctx->key_dma;
 346         }
 347
 348         desc = ctx->sh_desc_enc;
 349         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 350         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 351                                    desc_bytes(desc), ctx->dir);
 352
 353         /*
 354          * Job Descriptor and Shared Descriptors
 355          * must all fit into the 64-word Descriptor h/w Buffer
 356          */
 357         if (rem_bytes >= DESC_GCM_DEC_LEN) {
 358                 ctx->cdata.key_inline = true;
 359                 ctx->cdata.key_virt = ctx->key;
 360         } else {
 361                 ctx->cdata.key_inline = false;
 362                 ctx->cdata.key_dma = ctx->key_dma;
 363         }
 364
 365         desc = ctx->sh_desc_dec;
 366         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 367         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 368                                    desc_bytes(desc), ctx->dir);
 369
 370         return 0;
 371 }
 372
 373 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 374 {
 375         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 376         int err;
 377
 378         err = crypto_gcm_check_authsize(authsize);
 379         if (err)
 380                 return err;
 381
 382         ctx->authsize = authsize;
 383         gcm_set_sh_desc(authenc);
 384
 385         return 0;
 386 }
 387
 388 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 389 {
 390         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 391         struct device *jrdev = ctx->jrdev;
 392         unsigned int ivsize = crypto_aead_ivsize(aead);
 393         u32 *desc;
 394         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 395                         ctx->cdata.keylen;
 396
 397         if (!ctx->cdata.keylen || !ctx->authsize)
 398                 return 0;
 399
 400         /*
 401          * RFC4106 encrypt shared descriptor
 402          * Job Descriptor and Shared Descriptor
 403          * must fit into the 64-word Descriptor h/w Buffer
 404          */
 405         if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
 406                 ctx->cdata.key_inline = true;
 407                 ctx->cdata.key_virt = ctx->key;
 408         } else {
 409                 ctx->cdata.key_inline = false;
 410                 ctx->cdata.key_dma = ctx->key_dma;
 411         }
 412
 413         desc = ctx->sh_desc_enc;
 414         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 415                                   false);
 416         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 417                                    desc_bytes(desc), ctx->dir);
 418
 419         /*
 420          * Job Descriptor and Shared Descriptors
 421          * must all fit into the 64-word Descriptor h/w Buffer
 422          */
 423         if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
 424                 ctx->cdata.key_inline = true;
 425                 ctx->cdata.key_virt = ctx->key;
 426         } else {
 427                 ctx->cdata.key_inline = false;
 428                 ctx->cdata.key_dma = ctx->key_dma;
 429         }
 430
 431         desc = ctx->sh_desc_dec;
 432         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 433                                   false);
 434         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 435                                    desc_bytes(desc), ctx->dir);
 436
 437         return 0;
 438 }
 439
 440 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 441                                unsigned int authsize)
 442 {
 443         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 444         int err;
 445
 446         err = crypto_rfc4106_check_authsize(authsize);
 447         if (err)
 448                 return err;
 449
 450         ctx->authsize = authsize;
 451         rfc4106_set_sh_desc(authenc);
 452
 453         return 0;
 454 }
 455
 456 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 457 {
 458         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 459         struct device *jrdev = ctx->jrdev;
 460         unsigned int ivsize = crypto_aead_ivsize(aead);
 461         u32 *desc;
 462         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 463                         ctx->cdata.keylen;
 464
 465         if (!ctx->cdata.keylen || !ctx->authsize)
 466                 return 0;
 467
 468         /*
 469          * RFC4543 encrypt shared descriptor
 470          * Job Descriptor and Shared Descriptor
 471          * must fit into the 64-word Descriptor h/w Buffer
 472          */
 473         if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
 474                 ctx->cdata.key_inline = true;
 475                 ctx->cdata.key_virt = ctx->key;
 476         } else {
 477                 ctx->cdata.key_inline = false;
 478                 ctx->cdata.key_dma = ctx->key_dma;
 479         }
 480
 481         desc = ctx->sh_desc_enc;
 482         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 483                                   false);
 484         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 485                                    desc_bytes(desc), ctx->dir);
 486
 487         /*
 488          * Job Descriptor and Shared Descriptors
 489          * must all fit into the 64-word Descriptor h/w Buffer
 490          */
 491         if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
 492                 ctx->cdata.key_inline = true;
 493                 ctx->cdata.key_virt = ctx->key;
 494         } else {
 495                 ctx->cdata.key_inline = false;
 496                 ctx->cdata.key_dma = ctx->key_dma;
 497         }
 498
 499         desc = ctx->sh_desc_dec;
 500         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 501                                   false);
 502         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 503                                    desc_bytes(desc), ctx->dir);
 504
 505         return 0;
 506 }
 507
 508 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 509                                unsigned int authsize)
 510 {
 511         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 512
 513         if (authsize != 16)
 514                 return -EINVAL;
 515
 516         ctx->authsize = authsize;
 517         rfc4543_set_sh_desc(authenc);
 518
 519         return 0;
 520 }
 521
 522 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 523 {
 524         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 525         struct device *jrdev = ctx->jrdev;
 526         unsigned int ivsize = crypto_aead_ivsize(aead);
 527         u32 *desc;
 528
 529         if (!ctx->cdata.keylen || !ctx->authsize)
 530                 return 0;
 531
 532         desc = ctx->sh_desc_enc;
 533         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 534                                ctx->authsize, true, false);
 535         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 536                                    desc_bytes(desc), ctx->dir);
 537
 538         desc = ctx->sh_desc_dec;
 539         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 540                                ctx->authsize, false, false);
 541         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 542                                    desc_bytes(desc), ctx->dir);
 543
 544         return 0;
 545 }
 546
 547 static int chachapoly_setauthsize(struct crypto_aead *aead,
 548                                   unsigned int authsize)
 549 {
 550         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 551
 552         if (authsize != POLY1305_DIGEST_SIZE)
 553                 return -EINVAL;
 554
 555         ctx->authsize = authsize;
 556         return chachapoly_set_sh_desc(aead);
 557 }
 558
 559 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 560                              unsigned int keylen)
 561 {
 562         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 563         unsigned int ivsize = crypto_aead_ivsize(aead);
 564         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 565
 566         if (keylen != CHACHA_KEY_SIZE + saltlen)
 567                 return -EINVAL;
 568
 569         ctx->cdata.key_virt = key;
 570         ctx->cdata.keylen = keylen - saltlen;
 571
 572         return chachapoly_set_sh_desc(aead);
 573 }
 574
 575 static int aead_setkey(struct crypto_aead *aead,
 576                                const u8 *key, unsigned int keylen)
 577 {
 578         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 579         struct device *jrdev = ctx->jrdev;
 580         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 581         struct crypto_authenc_keys keys;
 582         int ret = 0;
 583
 584         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 585                 goto badkey;
 586
 587         dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 588                keys.authkeylen + keys.enckeylen, keys.enckeylen,
 589                keys.authkeylen);
 590         print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 591                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 592
 593         /*
 594          * If DKP is supported, use it in the shared descriptor to generate
 595          * the split key.
 596          */
 597         if (ctrlpriv->era >= 6) {
 598                 ctx->adata.keylen = keys.authkeylen;
 599                 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 600                                                       OP_ALG_ALGSEL_MASK);
 601
 602                 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 603                         goto badkey;
 604
 605                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
 606                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 607                        keys.enckeylen);
 608                 dma_sync_single_for_device(jrdev, ctx->key_dma,
 609                                            ctx->adata.keylen_pad +
 610                                            keys.enckeylen, ctx->dir);
 611                 goto skip_split_key;
 612         }
 613
 614         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
 615                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 616                             keys.enckeylen);
 617         if (ret) {
 618                 goto badkey;
 619         }
 620
 621         /* postpend encryption key to auth split key */
 622         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 623         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 624                                    keys.enckeylen, ctx->dir);
 625
 626         print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
 627                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 628                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 629
 630 skip_split_key:
 631         ctx->cdata.keylen = keys.enckeylen;
 632         memzero_explicit(&keys, sizeof(keys));
 633         return aead_set_sh_desc(aead);
 634 badkey:
 635         memzero_explicit(&keys, sizeof(keys));
 636         return -EINVAL;
 637 }
 638
 639 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 640                             unsigned int keylen)
 641 {
 642         struct crypto_authenc_keys keys;
 643         int err;
 644
 645         err = crypto_authenc_extractkeys(&keys, key, keylen);
 646         if (unlikely(err))
 647                 return err;
 648
 649         err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
 650               aead_setkey(aead, key, keylen);
 651
 652         memzero_explicit(&keys, sizeof(keys));
 653         return err;
 654 }
 655
 656 static int gcm_setkey(struct crypto_aead *aead,
 657                       const u8 *key, unsigned int keylen)
 658 {
 659         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 660         struct device *jrdev = ctx->jrdev;
 661         int err;
 662
 663         err = aes_check_keylen(keylen);
 664         if (err)
 665                 return err;
 666
 667         print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 668                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 669
 670         memcpy(ctx->key, key, keylen);
 671         dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
 672         ctx->cdata.keylen = keylen;
 673
 674         return gcm_set_sh_desc(aead);
 675 }
 676
 677 static int rfc4106_setkey(struct crypto_aead *aead,
 678                           const u8 *key, unsigned int keylen)
 679 {
 680         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 681         struct device *jrdev = ctx->jrdev;
 682         int err;
 683
 684         err = aes_check_keylen(keylen - 4);
 685         if (err)
 686                 return err;
 687
 688         print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 689                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 690
 691         memcpy(ctx->key, key, keylen);
 692
 693         /*
 694          * The last four bytes of the key material are used as the salt value
 695          * in the nonce. Update the AES key length.
 696          */
 697         ctx->cdata.keylen = keylen - 4;
 698         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 699                                    ctx->dir);
 700         return rfc4106_set_sh_desc(aead);
 701 }
 702
 703 static int rfc4543_setkey(struct crypto_aead *aead,
 704                           const u8 *key, unsigned int keylen)
 705 {
 706         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 707         struct device *jrdev = ctx->jrdev;
 708         int err;
 709
 710         err = aes_check_keylen(keylen - 4);
 711         if (err)
 712                 return err;
 713
 714         print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 715                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 716
 717         memcpy(ctx->key, key, keylen);
 718
 719         /*
 720          * The last four bytes of the key material are used as the salt value
 721          * in the nonce. Update the AES key length.
 722          */
 723         ctx->cdata.keylen = keylen - 4;
 724         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 725                                    ctx->dir);
 726         return rfc4543_set_sh_desc(aead);
 727 }
 728
 729 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 730                            unsigned int keylen, const u32 ctx1_iv_off)
 731 {
 732         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 733         struct caam_skcipher_alg *alg =
 734                 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 735                              skcipher);
 736         struct device *jrdev = ctx->jrdev;
 737         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 738         u32 *desc;
 739         const bool is_rfc3686 = alg->caam.rfc3686;
 740
 741         print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 742                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 743
 744         ctx->cdata.keylen = keylen;
 745         ctx->cdata.key_virt = key;
 746         ctx->cdata.key_inline = true;
 747
 748         /* skcipher_encrypt shared descriptor */
 749         desc = ctx->sh_desc_enc;
 750         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 751                                    ctx1_iv_off);
 752         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 753                                    desc_bytes(desc), ctx->dir);
 754
 755         /* skcipher_decrypt shared descriptor */
 756         desc = ctx->sh_desc_dec;
 757         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 758                                    ctx1_iv_off);
 759         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 760                                    desc_bytes(desc), ctx->dir);
 761
 762         return 0;
 763 }
 764
 765 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 766                                const u8 *key, unsigned int keylen)
 767 {
 768         int err;
 769
 770         err = aes_check_keylen(keylen);
 771         if (err)
 772                 return err;
 773
 774         return skcipher_setkey(skcipher, key, keylen, 0);
 775 }
 776
 777 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 778                                    const u8 *key, unsigned int keylen)
 779 {
 780         u32 ctx1_iv_off;
 781         int err;
 782
 783         /*
 784          * RFC3686 specific:
 785          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 786          *      | *key = {KEY, NONCE}
 787          */
 788         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 789         keylen -= CTR_RFC3686_NONCE_SIZE;
 790
 791         err = aes_check_keylen(keylen);
 792         if (err)
 793                 return err;
 794
 795         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 796 }
 797
 798 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
 799                                const u8 *key, unsigned int keylen)
 800 {
 801         u32 ctx1_iv_off;
 802         int err;
 803
 804         /*
 805          * AES-CTR needs to load IV in CONTEXT1 reg
 806          * at an offset of 128bits (16bytes)
 807          * CONTEXT1[255:128] = IV
 808          */
 809         ctx1_iv_off = 16;
 810
 811         err = aes_check_keylen(keylen);
 812         if (err)
 813                 return err;
 814
 815         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 816 }
 817
 818 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
 819                                const u8 *key, unsigned int keylen)
 820 {
 821         return verify_skcipher_des_key(skcipher, key) ?:
 822                skcipher_setkey(skcipher, key, keylen, 0);
 823 }
 824
 825 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
 826                                 const u8 *key, unsigned int keylen)
 827 {
 828         return verify_skcipher_des3_key(skcipher, key) ?:
 829                skcipher_setkey(skcipher, key, keylen, 0);
 830 }
 831
 832 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 833                                unsigned int keylen)
 834 {
 835         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 836         struct device *jrdev = ctx->jrdev;
 837         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 838         u32 *desc;
 839         int err;
 840
 841         err = xts_verify_key(skcipher, key, keylen);
 842         if (err) {
 843                 dev_dbg(jrdev, "key size mismatch\n");
 844                 return err;
 845         }
 846
 847         if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
 848                 ctx->xts_key_fallback = true;
 849
 850         if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
 851                 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
 852                 if (err)
 853                         return err;
 854         }
 855
 856         ctx->cdata.keylen = keylen;
 857         ctx->cdata.key_virt = key;
 858         ctx->cdata.key_inline = true;
 859
 860         /* xts_skcipher_encrypt shared descriptor */
 861         desc = ctx->sh_desc_enc;
 862         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
 863         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 864                                    desc_bytes(desc), ctx->dir);
 865
 866         /* xts_skcipher_decrypt shared descriptor */
 867         desc = ctx->sh_desc_dec;
 868         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
 869         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 870                                    desc_bytes(desc), ctx->dir);
 871
 872         return 0;
 873 }
 874
 875 /*
 876  * aead_edesc - s/w-extended aead descriptor
 877  * @src_nents: number of segments in input s/w scatterlist
 878  * @dst_nents: number of segments in output s/w scatterlist
 879  * @mapped_src_nents: number of segments in input h/w link table
 880  * @mapped_dst_nents: number of segments in output h/w link table
 881  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 882  * @bklog: stored to determine if the request needs backlog
 883  * @sec4_sg_dma: bus physical mapped address of h/w link table
 884  * @sec4_sg: pointer to h/w link table
 885  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 886  */
 887 struct aead_edesc {
 888         int src_nents;
 889         int dst_nents;
 890         int mapped_src_nents;
 891         int mapped_dst_nents;
 892         int sec4_sg_bytes;
 893         bool bklog;
 894         dma_addr_t sec4_sg_dma;
 895         struct sec4_sg_entry *sec4_sg;
 896         u32 hw_desc[];
 897 };
 898
 899 /*
 900  * skcipher_edesc - s/w-extended skcipher descriptor
 901  * @src_nents: number of segments in input s/w scatterlist
 902  * @dst_nents: number of segments in output s/w scatterlist
 903  * @mapped_src_nents: number of segments in input h/w link table
 904  * @mapped_dst_nents: number of segments in output h/w link table
 905  * @iv_dma: dma address of iv for checking continuity and link table
 906  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 907  * @bklog: stored to determine if the request needs backlog
 908  * @sec4_sg_dma: bus physical mapped address of h/w link table
 909  * @sec4_sg: pointer to h/w link table
 910  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 911  *           and IV
 912  */
 913 struct skcipher_edesc {
 914         int src_nents;
 915         int dst_nents;
 916         int mapped_src_nents;
 917         int mapped_dst_nents;
 918         dma_addr_t iv_dma;
 919         int sec4_sg_bytes;
 920         bool bklog;
 921         dma_addr_t sec4_sg_dma;
 922         struct sec4_sg_entry *sec4_sg;
 923         u32 hw_desc[];
 924 };
 925
 926 static void caam_unmap(struct device *dev, struct scatterlist *src,
 927                        struct scatterlist *dst, int src_nents,
 928                        int dst_nents,
 929                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 930                        int sec4_sg_bytes)
 931 {
 932         if (dst != src) {
 933                 if (src_nents)
 934                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 935                 if (dst_nents)
 936                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 937         } else {
 938                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 939         }
 940
 941         if (iv_dma)
 942                 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
 943         if (sec4_sg_bytes)
 944                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 945                                  DMA_TO_DEVICE);
 946 }
 947
 948 static void aead_unmap(struct device *dev,
 949                        struct aead_edesc *edesc,
 950                        struct aead_request *req)
 951 {
 952         caam_unmap(dev, req->src, req->dst,
 953                    edesc->src_nents, edesc->dst_nents, 0, 0,
 954                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 955 }
 956
 957 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 958                            struct skcipher_request *req)
 959 {
 960         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 961         int ivsize = crypto_skcipher_ivsize(skcipher);
 962
 963         caam_unmap(dev, req->src, req->dst,
 964                    edesc->src_nents, edesc->dst_nents,
 965                    edesc->iv_dma, ivsize,
 966                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 967 }
 968
 969 static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
 970                             void *context)
 971 {
 972         struct aead_request *req = context;
 973         struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
 974         struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
 975         struct aead_edesc *edesc;
 976         int ecode = 0;
 977         bool has_bklog;
 978
 979         dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 980
 981         edesc = rctx->edesc;
 982         has_bklog = edesc->bklog;
 983
 984         if (err)
 985                 ecode = caam_jr_strstatus(jrdev, err);
 986
 987         aead_unmap(jrdev, edesc, req);
 988
 989         kfree(edesc);
 990
 991         /*
 992          * If no backlog flag, the completion of the request is done
 993          * by CAAM, not crypto engine.
 994          */
 995         if (!has_bklog)
 996                 aead_request_complete(req, ecode);
 997         else
 998                 crypto_finalize_aead_request(jrp->engine, req, ecode);
 999 }
1000
1001 static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
1002                                 void *context)
1003 {
1004         struct skcipher_request *req = context;
1005         struct skcipher_edesc *edesc;
1006         struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1007         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1008         struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
1009         int ivsize = crypto_skcipher_ivsize(skcipher);
1010         int ecode = 0;
1011         bool has_bklog;
1012
1013         dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1014
1015         edesc = rctx->edesc;
1016         has_bklog = edesc->bklog;
1017         if (err)
1018                 ecode = caam_jr_strstatus(jrdev, err);
1019
1020         skcipher_unmap(jrdev, edesc, req);
1021
1022         /*
1023          * The crypto API expects us to set the IV (req->iv) to the last
1024          * ciphertext block (CBC mode) or last counter (CTR mode).
1025          * This is used e.g. by the CTS mode.
1026          */
1027         if (ivsize && !ecode) {
1028                 memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
1029                        ivsize);
1030
1031                 print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1032                                      DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1033                                      ivsize, 1);
1034         }
1035
1036         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1037                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1038                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1039
1040         kfree(edesc);
1041
1042         /*
1043          * If no backlog flag, the completion of the request is done
1044          * by CAAM, not crypto engine.
1045          */
1046         if (!has_bklog)
1047                 skcipher_request_complete(req, ecode);
1048         else
1049                 crypto_finalize_skcipher_request(jrp->engine, req, ecode);
1050 }
1051
1052 /*
1053  * Fill in aead job descriptor
1054  */
1055 static void init_aead_job(struct aead_request *req,
1056                           struct aead_edesc *edesc,
1057                           bool all_contig, bool encrypt)
1058 {
1059         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1060         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1061         int authsize = ctx->authsize;
1062         u32 *desc = edesc->hw_desc;
1063         u32 out_options, in_options;
1064         dma_addr_t dst_dma, src_dma;
1065         int len, sec4_sg_index = 0;
1066         dma_addr_t ptr;
1067         u32 *sh_desc;
1068
1069         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1070         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1071
1072         len = desc_len(sh_desc);
1073         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1074
1075         if (all_contig) {
1076                 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
1077                                                     0;
1078                 in_options = 0;
1079         } else {
1080                 src_dma = edesc->sec4_sg_dma;
1081                 sec4_sg_index += edesc->mapped_src_nents;
1082                 in_options = LDST_SGF;
1083         }
1084
1085         append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1086                           in_options);
1087
1088         dst_dma = src_dma;
1089         out_options = in_options;
1090
1091         if (unlikely(req->src != req->dst)) {
1092                 if (!edesc->mapped_dst_nents) {
1093                         dst_dma = 0;
1094                         out_options = 0;
1095                 } else if (edesc->mapped_dst_nents == 1) {
1096                         dst_dma = sg_dma_address(req->dst);
1097                         out_options = 0;
1098                 } else {
1099                         dst_dma = edesc->sec4_sg_dma +
1100                                   sec4_sg_index *
1101                                   sizeof(struct sec4_sg_entry);
1102                         out_options = LDST_SGF;
1103                 }
1104         }
1105
1106         if (encrypt)
1107                 append_seq_out_ptr(desc, dst_dma,
1108                                    req->assoclen + req->cryptlen + authsize,
1109                                    out_options);
1110         else
1111                 append_seq_out_ptr(desc, dst_dma,
1112                                    req->assoclen + req->cryptlen - authsize,
1113                                    out_options);
1114 }
1115
1116 static void init_gcm_job(struct aead_request *req,
1117                          struct aead_edesc *edesc,
1118                          bool all_contig, bool encrypt)
1119 {
1120         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1121         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1122         unsigned int ivsize = crypto_aead_ivsize(aead);
1123         u32 *desc = edesc->hw_desc;
1124         bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
1125         unsigned int last;
1126
1127         init_aead_job(req, edesc, all_contig, encrypt);
1128         append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1129
1130         /* BUG This should not be specific to generic GCM. */
1131         last = 0;
1132         if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1133                 last = FIFOLD_TYPE_LAST1;
1134
1135         /* Read GCM IV */
1136         append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1137                          FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
1138         /* Append Salt */
1139         if (!generic_gcm)
1140                 append_data(desc, ctx->key + ctx->cdata.keylen, 4);
1141         /* Append IV */
1142         append_data(desc, req->iv, ivsize);
1143         /* End of blank commands */
1144 }
1145
1146 static void init_chachapoly_job(struct aead_request *req,
1147                                 struct aead_edesc *edesc, bool all_contig,
1148                                 bool encrypt)
1149 {
1150         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1151         unsigned int ivsize = crypto_aead_ivsize(aead);
1152         unsigned int assoclen = req->assoclen;
1153         u32 *desc = edesc->hw_desc;
1154         u32 ctx_iv_off = 4;
1155
1156         init_aead_job(req, edesc, all_contig, encrypt);
1157
1158         if (ivsize != CHACHAPOLY_IV_SIZE) {
1159                 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1160                 ctx_iv_off += 4;
1161
1162                 /*
1163                  * The associated data comes already with the IV but we need
1164                  * to skip it when we authenticate or encrypt...
1165                  */
1166                 assoclen -= ivsize;
1167         }
1168
1169         append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1170
1171         /*
1172          * For IPsec load the IV further in the same register.
1173          * For RFC7539 simply load the 12 bytes nonce in a single operation
1174          */
1175         append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
1176                            LDST_SRCDST_BYTE_CONTEXT |
1177                            ctx_iv_off << LDST_OFFSET_SHIFT);
1178 }
1179
1180 static void init_authenc_job(struct aead_request *req,
1181                              struct aead_edesc *edesc,
1182                              bool all_contig, bool encrypt)
1183 {
1184         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1185         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1186                                                  struct caam_aead_alg, aead);
1187         unsigned int ivsize = crypto_aead_ivsize(aead);
1188         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1189         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1190         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1191                                OP_ALG_AAI_CTR_MOD128);
1192         const bool is_rfc3686 = alg->caam.rfc3686;
1193         u32 *desc = edesc->hw_desc;
1194         u32 ivoffset = 0;
1195
1196         /*
1197          * AES-CTR needs to load IV in CONTEXT1 reg
1198          * at an offset of 128bits (16bytes)
1199          * CONTEXT1[255:128] = IV
1200          */
1201         if (ctr_mode)
1202                 ivoffset = 16;
1203
1204         /*
1205          * RFC3686 specific:
1206          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1207          */
1208         if (is_rfc3686)
1209                 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1210
1211         init_aead_job(req, edesc, all_contig, encrypt);
1212
1213         /*
1214          * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1215          * having DPOVRD as destination.
1216          */
1217         if (ctrlpriv->era < 3)
1218                 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1219         else
1220                 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1221
1222         if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1223                 append_load_as_imm(desc, req->iv, ivsize,
1224                                    LDST_CLASS_1_CCB |
1225                                    LDST_SRCDST_BYTE_CONTEXT |
1226                                    (ivoffset << LDST_OFFSET_SHIFT));
1227 }
1228
1229 /*
1230  * Fill in skcipher job descriptor
1231  */
1232 static void init_skcipher_job(struct skcipher_request *req,
1233                               struct skcipher_edesc *edesc,
1234                               const bool encrypt)
1235 {
1236         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1237         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1238         struct device *jrdev = ctx->jrdev;
1239         int ivsize = crypto_skcipher_ivsize(skcipher);
1240         u32 *desc = edesc->hw_desc;
1241         u32 *sh_desc;
1242         u32 in_options = 0, out_options = 0;
1243         dma_addr_t src_dma, dst_dma, ptr;
1244         int len, sec4_sg_index = 0;
1245
1246         print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
1247                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1248         dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
1249                (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
1250
1251         caam_dump_sg("src    @" __stringify(__LINE__)": ",
1252                      DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1253                      edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1254
1255         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1256         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1257
1258         len = desc_len(sh_desc);
1259         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1260
1261         if (ivsize || edesc->mapped_src_nents > 1) {
1262                 src_dma = edesc->sec4_sg_dma;
1263                 sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
1264                 in_options = LDST_SGF;
1265         } else {
1266                 src_dma = sg_dma_address(req->src);
1267         }
1268
1269         append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
1270
1271         if (likely(req->src == req->dst)) {
1272                 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
1273                 out_options = in_options;
1274         } else if (!ivsize && edesc->mapped_dst_nents == 1) {
1275                 dst_dma = sg_dma_address(req->dst);
1276         } else {
1277                 dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
1278                           sizeof(struct sec4_sg_entry);
1279                 out_options = LDST_SGF;
1280         }
1281
1282         append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
1283 }
1284
1285 /*
1286  * allocate and map the aead extended descriptor
1287  */
1288 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1289                                            int desc_bytes, bool *all_contig_ptr,
1290                                            bool encrypt)
1291 {
1292         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1293         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1294         struct device *jrdev = ctx->jrdev;
1295         struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1296         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1297                        GFP_KERNEL : GFP_ATOMIC;
1298         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1299         int src_len, dst_len = 0;
1300         struct aead_edesc *edesc;
1301         int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1302         unsigned int authsize = ctx->authsize;
1303
1304         if (unlikely(req->dst != req->src)) {
1305                 src_len = req->assoclen + req->cryptlen;
1306                 dst_len = src_len + (encrypt ? authsize : (-authsize));
1307
1308                 src_nents = sg_nents_for_len(req->src, src_len);
1309                 if (unlikely(src_nents < 0)) {
1310                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1311                                 src_len);
1312                         return ERR_PTR(src_nents);
1313                 }
1314
1315                 dst_nents = sg_nents_for_len(req->dst, dst_len);
1316                 if (unlikely(dst_nents < 0)) {
1317                         dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1318                                 dst_len);
1319                         return ERR_PTR(dst_nents);
1320                 }
1321         } else {
1322                 src_len = req->assoclen + req->cryptlen +
1323                           (encrypt ? authsize : 0);
1324
1325                 src_nents = sg_nents_for_len(req->src, src_len);
1326                 if (unlikely(src_nents < 0)) {
1327                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1328                                 src_len);
1329                         return ERR_PTR(src_nents);
1330                 }
1331         }
1332
1333         if (likely(req->src == req->dst)) {
1334                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1335                                               DMA_BIDIRECTIONAL);
1336                 if (unlikely(!mapped_src_nents)) {
1337                         dev_err(jrdev, "unable to map source\n");
1338                         return ERR_PTR(-ENOMEM);
1339                 }
1340         } else {
1341                 /* Cover also the case of null (zero length) input data */
1342                 if (src_nents) {
1343                         mapped_src_nents = dma_map_sg(jrdev, req->src,
1344                                                       src_nents, DMA_TO_DEVICE);
1345                         if (unlikely(!mapped_src_nents)) {
1346                                 dev_err(jrdev, "unable to map source\n");
1347                                 return ERR_PTR(-ENOMEM);
1348                         }
1349                 } else {
1350                         mapped_src_nents = 0;
1351                 }
1352
1353                 /* Cover also the case of null (zero length) output data */
1354                 if (dst_nents) {
1355                         mapped_dst_nents = dma_map_sg(jrdev, req->dst,
1356                                                       dst_nents,
1357                                                       DMA_FROM_DEVICE);
1358                         if (unlikely(!mapped_dst_nents)) {
1359                                 dev_err(jrdev, "unable to map destination\n");
1360                                 dma_unmap_sg(jrdev, req->src, src_nents,
1361                                              DMA_TO_DEVICE);
1362                                 return ERR_PTR(-ENOMEM);
1363                         }
1364                 } else {
1365                         mapped_dst_nents = 0;
1366                 }
1367         }
1368
1369         /*
1370          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1371          * the end of the table by allocating more S/G entries.
1372          */
1373         sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1374         if (mapped_dst_nents > 1)
1375                 sec4_sg_len += pad_sg_nents(mapped_dst_nents);
1376         else
1377                 sec4_sg_len = pad_sg_nents(sec4_sg_len);
1378
1379         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1380
1381         /* allocate space for base edesc and hw desc commands, link tables */
1382         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1383                         GFP_DMA | flags);
1384         if (!edesc) {
1385                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1386                            0, 0, 0);
1387                 return ERR_PTR(-ENOMEM);
1388         }
1389
1390         edesc->src_nents = src_nents;
1391         edesc->dst_nents = dst_nents;
1392         edesc->mapped_src_nents = mapped_src_nents;
1393         edesc->mapped_dst_nents = mapped_dst_nents;
1394         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1395                          desc_bytes;
1396
1397         rctx->edesc = edesc;
1398
1399         *all_contig_ptr = !(mapped_src_nents > 1);
1400
1401         sec4_sg_index = 0;
1402         if (mapped_src_nents > 1) {
1403                 sg_to_sec4_sg_last(req->src, src_len,
1404                                    edesc->sec4_sg + sec4_sg_index, 0);
1405                 sec4_sg_index += mapped_src_nents;
1406         }
1407         if (mapped_dst_nents > 1) {
1408                 sg_to_sec4_sg_last(req->dst, dst_len,
1409                                    edesc->sec4_sg + sec4_sg_index, 0);
1410         }
1411
1412         if (!sec4_sg_bytes)
1413                 return edesc;
1414
1415         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1416                                             sec4_sg_bytes, DMA_TO_DEVICE);
1417         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1418                 dev_err(jrdev, "unable to map S/G table\n");
1419                 aead_unmap(jrdev, edesc, req);
1420                 kfree(edesc);
1421                 return ERR_PTR(-ENOMEM);
1422         }
1423
1424         edesc->sec4_sg_bytes = sec4_sg_bytes;
1425
1426         return edesc;
1427 }
1428
1429 static int aead_enqueue_req(struct device *jrdev, struct aead_request *req)
1430 {
1431         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
1432         struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1433         struct aead_edesc *edesc = rctx->edesc;
1434         u32 *desc = edesc->hw_desc;
1435         int ret;
1436
1437         /*
1438          * Only the backlog request are sent to crypto-engine since the others
1439          * can be handled by CAAM, if free, especially since JR has up to 1024
1440          * entries (more than the 10 entries from crypto-engine).
1441          */
1442         if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1443                 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine,
1444                                                              req);
1445         else
1446                 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req);
1447
1448         if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1449                 aead_unmap(jrdev, edesc, req);
1450                 kfree(rctx->edesc);
1451         }
1452
1453         return ret;
1454 }
1455
1456 static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
1457 {
1458         struct aead_edesc *edesc;
1459         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1460         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1461         struct device *jrdev = ctx->jrdev;
1462         bool all_contig;
1463         u32 *desc;
1464
1465         edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
1466                                  encrypt);
1467         if (IS_ERR(edesc))
1468                 return PTR_ERR(edesc);
1469
1470         desc = edesc->hw_desc;
1471
1472         init_chachapoly_job(req, edesc, all_contig, encrypt);
1473         print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1474                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1475                              1);
1476
1477         return aead_enqueue_req(jrdev, req);
1478 }
1479
1480 static int chachapoly_encrypt(struct aead_request *req)
1481 {
1482         return chachapoly_crypt(req, true);
1483 }
1484
1485 static int chachapoly_decrypt(struct aead_request *req)
1486 {
1487         return chachapoly_crypt(req, false);
1488 }
1489
1490 static inline int aead_crypt(struct aead_request *req, bool encrypt)
1491 {
1492         struct aead_edesc *edesc;
1493         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1494         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1495         struct device *jrdev = ctx->jrdev;
1496         bool all_contig;
1497
1498         /* allocate extended descriptor */
1499         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1500                                  &all_contig, encrypt);
1501         if (IS_ERR(edesc))
1502                 return PTR_ERR(edesc);
1503
1504         /* Create and submit job descriptor */
1505         init_authenc_job(req, edesc, all_contig, encrypt);
1506
1507         print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1508                              DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1509                              desc_bytes(edesc->hw_desc), 1);
1510
1511         return aead_enqueue_req(jrdev, req);
1512 }
1513
1514 static int aead_encrypt(struct aead_request *req)
1515 {
1516         return aead_crypt(req, true);
1517 }
1518
1519 static int aead_decrypt(struct aead_request *req)
1520 {
1521         return aead_crypt(req, false);
1522 }
1523
1524 static int aead_do_one_req(struct crypto_engine *engine, void *areq)
1525 {
1526         struct aead_request *req = aead_request_cast(areq);
1527         struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1528         struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1529         u32 *desc = rctx->edesc->hw_desc;
1530         int ret;
1531
1532         rctx->edesc->bklog = true;
1533
1534         ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
1535
1536         if (ret == -ENOSPC && engine->retry_support)
1537                 return ret;
1538
1539         if (ret != -EINPROGRESS) {
1540                 aead_unmap(ctx->jrdev, rctx->edesc, req);
1541                 kfree(rctx->edesc);
1542         } else {
1543                 ret = 0;
1544         }
1545
1546         return ret;
1547 }
1548
1549 static inline int gcm_crypt(struct aead_request *req, bool encrypt)
1550 {
1551         struct aead_edesc *edesc;
1552         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1553         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1554         struct device *jrdev = ctx->jrdev;
1555         bool all_contig;
1556
1557         /* allocate extended descriptor */
1558         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig,
1559                                  encrypt);
1560         if (IS_ERR(edesc))
1561                 return PTR_ERR(edesc);
1562
1563         /* Create and submit job descriptor */
1564         init_gcm_job(req, edesc, all_contig, encrypt);
1565
1566         print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1567                              DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1568                              desc_bytes(edesc->hw_desc), 1);
1569
1570         return aead_enqueue_req(jrdev, req);
1571 }
1572
1573 static int gcm_encrypt(struct aead_request *req)
1574 {
1575         return gcm_crypt(req, true);
1576 }
1577
1578 static int gcm_decrypt(struct aead_request *req)
1579 {
1580         return gcm_crypt(req, false);
1581 }
1582
1583 static int ipsec_gcm_encrypt(struct aead_request *req)
1584 {
1585         return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
1586 }
1587
1588 static int ipsec_gcm_decrypt(struct aead_request *req)
1589 {
1590         return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
1591 }
1592
1593 /*
1594  * allocate and map the skcipher extended descriptor for skcipher
1595  */
1596 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1597                                                    int desc_bytes)
1598 {
1599         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1600         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1601         struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1602         struct device *jrdev = ctx->jrdev;
1603         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1604                        GFP_KERNEL : GFP_ATOMIC;
1605         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1606         struct skcipher_edesc *edesc;
1607         dma_addr_t iv_dma = 0;
1608         u8 *iv;
1609         int ivsize = crypto_skcipher_ivsize(skcipher);
1610         int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1611
1612         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1613         if (unlikely(src_nents < 0)) {
1614                 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1615                         req->cryptlen);
1616                 return ERR_PTR(src_nents);
1617         }
1618
1619         if (req->dst != req->src) {
1620                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1621                 if (unlikely(dst_nents < 0)) {
1622                         dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1623                                 req->cryptlen);
1624                         return ERR_PTR(dst_nents);
1625                 }
1626         }
1627
1628         if (likely(req->src == req->dst)) {
1629                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1630                                               DMA_BIDIRECTIONAL);
1631                 if (unlikely(!mapped_src_nents)) {
1632                         dev_err(jrdev, "unable to map source\n");
1633                         return ERR_PTR(-ENOMEM);
1634                 }
1635         } else {
1636                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1637                                               DMA_TO_DEVICE);
1638                 if (unlikely(!mapped_src_nents)) {
1639                         dev_err(jrdev, "unable to map source\n");
1640                         return ERR_PTR(-ENOMEM);
1641                 }
1642                 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1643                                               DMA_FROM_DEVICE);
1644                 if (unlikely(!mapped_dst_nents)) {
1645                         dev_err(jrdev, "unable to map destination\n");
1646                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1647                         return ERR_PTR(-ENOMEM);
1648                 }
1649         }
1650
1651         if (!ivsize && mapped_src_nents == 1)
1652                 sec4_sg_ents = 0; // no need for an input hw s/g table
1653         else
1654                 sec4_sg_ents = mapped_src_nents + !!ivsize;
1655         dst_sg_idx = sec4_sg_ents;
1656
1657         /*
1658          * Input, output HW S/G tables: [IV, src][dst, IV]
1659          * IV entries point to the same buffer
1660          * If src == dst, S/G entries are reused (S/G tables overlap)
1661          *
1662          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1663          * the end of the table by allocating more S/G entries. Logic:
1664          * if (output S/G)
1665          *      pad output S/G, if needed
1666          * else if (input S/G) ...
1667          *      pad input S/G, if needed
1668          */
1669         if (ivsize || mapped_dst_nents > 1) {
1670                 if (req->src == req->dst)
1671                         sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
1672                 else
1673                         sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
1674                                                      !!ivsize);
1675         } else {
1676                 sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
1677         }
1678
1679         sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1680
1681         /*
1682          * allocate space for base edesc and hw desc commands, link tables, IV
1683          */
1684         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
1685                         GFP_DMA | flags);
1686         if (!edesc) {
1687                 dev_err(jrdev, "could not allocate extended descriptor\n");
1688                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1689                            0, 0, 0);
1690                 return ERR_PTR(-ENOMEM);
1691         }
1692
1693         edesc->src_nents = src_nents;
1694         edesc->dst_nents = dst_nents;
1695         edesc->mapped_src_nents = mapped_src_nents;
1696         edesc->mapped_dst_nents = mapped_dst_nents;
1697         edesc->sec4_sg_bytes = sec4_sg_bytes;
1698         edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
1699                                                   desc_bytes);
1700         rctx->edesc = edesc;
1701
1702         /* Make sure IV is located in a DMAable area */
1703         if (ivsize) {
1704                 iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
1705                 memcpy(iv, req->iv, ivsize);
1706
1707                 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
1708                 if (dma_mapping_error(jrdev, iv_dma)) {
1709                         dev_err(jrdev, "unable to map IV\n");
1710                         caam_unmap(jrdev, req->src, req->dst, src_nents,
1711                                    dst_nents, 0, 0, 0, 0);
1712                         kfree(edesc);
1713                         return ERR_PTR(-ENOMEM);
1714                 }
1715
1716                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1717         }
1718         if (dst_sg_idx)
1719                 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
1720                               !!ivsize, 0);
1721
1722         if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
1723                 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
1724                               dst_sg_idx, 0);
1725
1726         if (ivsize)
1727                 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
1728                                    mapped_dst_nents, iv_dma, ivsize, 0);
1729
1730         if (ivsize || mapped_dst_nents > 1)
1731                 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
1732                                     mapped_dst_nents - 1 + !!ivsize);
1733
1734         if (sec4_sg_bytes) {
1735                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1736                                                     sec4_sg_bytes,
1737                                                     DMA_TO_DEVICE);
1738                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1739                         dev_err(jrdev, "unable to map S/G table\n");
1740                         caam_unmap(jrdev, req->src, req->dst, src_nents,
1741                                    dst_nents, iv_dma, ivsize, 0, 0);
1742                         kfree(edesc);
1743                         return ERR_PTR(-ENOMEM);
1744                 }
1745         }
1746
1747         edesc->iv_dma = iv_dma;
1748
1749         print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
1750                              DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1751                              sec4_sg_bytes, 1);
1752
1753         return edesc;
1754 }
1755
1756 static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
1757 {
1758         struct skcipher_request *req = skcipher_request_cast(areq);
1759         struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1760         struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1761         u32 *desc = rctx->edesc->hw_desc;
1762         int ret;
1763
1764         rctx->edesc->bklog = true;
1765
1766         ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
1767
1768         if (ret == -ENOSPC && engine->retry_support)
1769                 return ret;
1770
1771         if (ret != -EINPROGRESS) {
1772                 skcipher_unmap(ctx->jrdev, rctx->edesc, req);
1773                 kfree(rctx->edesc);
1774         } else {
1775                 ret = 0;
1776         }
1777
1778         return ret;
1779 }
1780
1781 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1782 {
1783         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1784         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1785
1786         return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1787 }
1788
1789 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1790 {
1791         struct skcipher_edesc *edesc;
1792         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1793         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1794         struct device *jrdev = ctx->jrdev;
1795         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
1796         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
1797         u32 *desc;
1798         int ret = 0;
1799
1800         /*
1801          * XTS is expected to return an error even for input length = 0
1802          * Note that the case input length < block size will be caught during
1803          * HW offloading and return an error.
1804          */
1805         if (!req->cryptlen && !ctx->fallback)
1806                 return 0;
1807
1808         if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
1809                               ctx->xts_key_fallback)) {
1810                 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1811
1812                 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
1813                 skcipher_request_set_callback(&rctx->fallback_req,
1814                                               req->base.flags,
1815                                               req->base.complete,
1816                                               req->base.data);
1817                 skcipher_request_set_crypt(&rctx->fallback_req, req->src,
1818                                            req->dst, req->cryptlen, req->iv);
1819
1820                 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
1821                                  crypto_skcipher_decrypt(&rctx->fallback_req);
1822         }
1823
1824         /* allocate extended descriptor */
1825         edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
1826         if (IS_ERR(edesc))
1827                 return PTR_ERR(edesc);
1828
1829         /* Create and submit job descriptor*/
1830         init_skcipher_job(req, edesc, encrypt);
1831
1832         print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
1833                              DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1834                              desc_bytes(edesc->hw_desc), 1);
1835
1836         desc = edesc->hw_desc;
1837         /*
1838          * Only the backlog request are sent to crypto-engine since the others
1839          * can be handled by CAAM, if free, especially since JR has up to 1024
1840          * entries (more than the 10 entries from crypto-engine).
1841          */
1842         if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1843                 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine,
1844                                                                  req);
1845         else
1846                 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req);
1847
1848         if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1849                 skcipher_unmap(jrdev, edesc, req);
1850                 kfree(edesc);
1851         }
1852
1853         return ret;
1854 }
1855
1856 static int skcipher_encrypt(struct skcipher_request *req)
1857 {
1858         return skcipher_crypt(req, true);
1859 }
1860
1861 static int skcipher_decrypt(struct skcipher_request *req)
1862 {
1863         return skcipher_crypt(req, false);
1864 }
1865
1866 static struct caam_skcipher_alg driver_algs[] = {
1867         {
1868                 .skcipher = {
1869                         .base = {
1870                                 .cra_name = "cbc(aes)",
1871                                 .cra_driver_name = "cbc-aes-caam",
1872                                 .cra_blocksize = AES_BLOCK_SIZE,
1873                         },
1874                         .setkey = aes_skcipher_setkey,
1875                         .encrypt = skcipher_encrypt,
1876                         .decrypt = skcipher_decrypt,
1877                         .min_keysize = AES_MIN_KEY_SIZE,
1878                         .max_keysize = AES_MAX_KEY_SIZE,
1879                         .ivsize = AES_BLOCK_SIZE,
1880                 },
1881                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1882         },
1883         {
1884                 .skcipher = {
1885                         .base = {
1886                                 .cra_name = "cbc(des3_ede)",
1887                                 .cra_driver_name = "cbc-3des-caam",
1888                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1889                         },
1890                         .setkey = des3_skcipher_setkey,
1891                         .encrypt = skcipher_encrypt,
1892                         .decrypt = skcipher_decrypt,
1893                         .min_keysize = DES3_EDE_KEY_SIZE,
1894                         .max_keysize = DES3_EDE_KEY_SIZE,
1895                         .ivsize = DES3_EDE_BLOCK_SIZE,
1896                 },
1897                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1898         },
1899         {
1900                 .skcipher = {
1901                         .base = {
1902                                 .cra_name = "cbc(des)",
1903                                 .cra_driver_name = "cbc-des-caam",
1904                                 .cra_blocksize = DES_BLOCK_SIZE,
1905                         },
1906                         .setkey = des_skcipher_setkey,
1907                         .encrypt = skcipher_encrypt,
1908                         .decrypt = skcipher_decrypt,
1909                         .min_keysize = DES_KEY_SIZE,
1910                         .max_keysize = DES_KEY_SIZE,
1911                         .ivsize = DES_BLOCK_SIZE,
1912                 },
1913                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1914         },
1915         {
1916                 .skcipher = {
1917                         .base = {
1918                                 .cra_name = "ctr(aes)",
1919                                 .cra_driver_name = "ctr-aes-caam",
1920                                 .cra_blocksize = 1,
1921                         },
1922                         .setkey = ctr_skcipher_setkey,
1923                         .encrypt = skcipher_encrypt,
1924                         .decrypt = skcipher_decrypt,
1925                         .min_keysize = AES_MIN_KEY_SIZE,
1926                         .max_keysize = AES_MAX_KEY_SIZE,
1927                         .ivsize = AES_BLOCK_SIZE,
1928                         .chunksize = AES_BLOCK_SIZE,
1929                 },
1930                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1931                                         OP_ALG_AAI_CTR_MOD128,
1932         },
1933         {
1934                 .skcipher = {
1935                         .base = {
1936                                 .cra_name = "rfc3686(ctr(aes))",
1937                                 .cra_driver_name = "rfc3686-ctr-aes-caam",
1938                                 .cra_blocksize = 1,
1939                         },
1940                         .setkey = rfc3686_skcipher_setkey,
1941                         .encrypt = skcipher_encrypt,
1942                         .decrypt = skcipher_decrypt,
1943                         .min_keysize = AES_MIN_KEY_SIZE +
1944                                        CTR_RFC3686_NONCE_SIZE,
1945                         .max_keysize = AES_MAX_KEY_SIZE +
1946                                        CTR_RFC3686_NONCE_SIZE,
1947                         .ivsize = CTR_RFC3686_IV_SIZE,
1948                         .chunksize = AES_BLOCK_SIZE,
1949                 },
1950                 .caam = {
1951                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1952                                            OP_ALG_AAI_CTR_MOD128,
1953                         .rfc3686 = true,
1954                 },
1955         },
1956         {
1957                 .skcipher = {
1958                         .base = {
1959                                 .cra_name = "xts(aes)",
1960                                 .cra_driver_name = "xts-aes-caam",
1961                                 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1962                                 .cra_blocksize = AES_BLOCK_SIZE,
1963                         },
1964                         .setkey = xts_skcipher_setkey,
1965                         .encrypt = skcipher_encrypt,
1966                         .decrypt = skcipher_decrypt,
1967                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1968                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1969                         .ivsize = AES_BLOCK_SIZE,
1970                 },
1971                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1972         },
1973         {
1974                 .skcipher = {
1975                         .base = {
1976                                 .cra_name = "ecb(des)",
1977                                 .cra_driver_name = "ecb-des-caam",
1978                                 .cra_blocksize = DES_BLOCK_SIZE,
1979                         },
1980                         .setkey = des_skcipher_setkey,
1981                         .encrypt = skcipher_encrypt,
1982                         .decrypt = skcipher_decrypt,
1983                         .min_keysize = DES_KEY_SIZE,
1984                         .max_keysize = DES_KEY_SIZE,
1985                 },
1986                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
1987         },
1988         {
1989                 .skcipher = {
1990                         .base = {
1991                                 .cra_name = "ecb(aes)",
1992                                 .cra_driver_name = "ecb-aes-caam",
1993                                 .cra_blocksize = AES_BLOCK_SIZE,
1994                         },
1995                         .setkey = aes_skcipher_setkey,
1996                         .encrypt = skcipher_encrypt,
1997                         .decrypt = skcipher_decrypt,
1998                         .min_keysize = AES_MIN_KEY_SIZE,
1999                         .max_keysize = AES_MAX_KEY_SIZE,
2000                 },
2001                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
2002         },
2003         {
2004                 .skcipher = {
2005                         .base = {
2006                                 .cra_name = "ecb(des3_ede)",
2007                                 .cra_driver_name = "ecb-des3-caam",
2008                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2009                         },
2010                         .setkey = des3_skcipher_setkey,
2011                         .encrypt = skcipher_encrypt,
2012                         .decrypt = skcipher_decrypt,
2013                         .min_keysize = DES3_EDE_KEY_SIZE,
2014                         .max_keysize = DES3_EDE_KEY_SIZE,
2015                 },
2016                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
2017         },
2018 };
2019
2020 static struct caam_aead_alg driver_aeads[] = {
2021         {
2022                 .aead = {
2023                         .base = {
2024                                 .cra_name = "rfc4106(gcm(aes))",
2025                                 .cra_driver_name = "rfc4106-gcm-aes-caam",
2026                                 .cra_blocksize = 1,
2027                         },
2028                         .setkey = rfc4106_setkey,
2029                         .setauthsize = rfc4106_setauthsize,
2030                         .encrypt = ipsec_gcm_encrypt,
2031                         .decrypt = ipsec_gcm_decrypt,
2032                         .ivsize = GCM_RFC4106_IV_SIZE,
2033                         .maxauthsize = AES_BLOCK_SIZE,
2034                 },
2035                 .caam = {
2036                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2037                         .nodkp = true,
2038                 },
2039         },
2040         {
2041                 .aead = {
2042                         .base = {
2043                                 .cra_name = "rfc4543(gcm(aes))",
2044                                 .cra_driver_name = "rfc4543-gcm-aes-caam",
2045                                 .cra_blocksize = 1,
2046                         },
2047                         .setkey = rfc4543_setkey,
2048                         .setauthsize = rfc4543_setauthsize,
2049                         .encrypt = ipsec_gcm_encrypt,
2050                         .decrypt = ipsec_gcm_decrypt,
2051                         .ivsize = GCM_RFC4543_IV_SIZE,
2052                         .maxauthsize = AES_BLOCK_SIZE,
2053                 },
2054                 .caam = {
2055                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2056                         .nodkp = true,
2057                 },
2058         },
2059         /* Galois Counter Mode */
2060         {
2061                 .aead = {
2062                         .base = {
2063                                 .cra_name = "gcm(aes)",
2064                                 .cra_driver_name = "gcm-aes-caam",
2065                                 .cra_blocksize = 1,
2066                         },
2067                         .setkey = gcm_setkey,
2068                         .setauthsize = gcm_setauthsize,
2069                         .encrypt = gcm_encrypt,
2070                         .decrypt = gcm_decrypt,
2071                         .ivsize = GCM_AES_IV_SIZE,
2072                         .maxauthsize = AES_BLOCK_SIZE,
2073                 },
2074                 .caam = {
2075                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2076                         .nodkp = true,
2077                 },
2078         },
2079         /* single-pass ipsec_esp descriptor */
2080         {
2081                 .aead = {
2082                         .base = {
2083                                 .cra_name = "authenc(hmac(md5),"
2084                                             "ecb(cipher_null))",
2085                                 .cra_driver_name = "authenc-hmac-md5-"
2086                                                    "ecb-cipher_null-caam",
2087                                 .cra_blocksize = NULL_BLOCK_SIZE,
2088                         },
2089                         .setkey = aead_setkey,
2090                         .setauthsize = aead_setauthsize,
2091                         .encrypt = aead_encrypt,
2092                         .decrypt = aead_decrypt,
2093                         .ivsize = NULL_IV_SIZE,
2094                         .maxauthsize = MD5_DIGEST_SIZE,
2095                 },
2096                 .caam = {
2097                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2098                                            OP_ALG_AAI_HMAC_PRECOMP,
2099                 },
2100         },
2101         {
2102                 .aead = {
2103                         .base = {
2104                                 .cra_name = "authenc(hmac(sha1),"
2105                                             "ecb(cipher_null))",
2106                                 .cra_driver_name = "authenc-hmac-sha1-"
2107                                                    "ecb-cipher_null-caam",
2108                                 .cra_blocksize = NULL_BLOCK_SIZE,
2109                         },
2110                         .setkey = aead_setkey,
2111                         .setauthsize = aead_setauthsize,
2112                         .encrypt = aead_encrypt,
2113                         .decrypt = aead_decrypt,
2114                         .ivsize = NULL_IV_SIZE,
2115                         .maxauthsize = SHA1_DIGEST_SIZE,
2116                 },
2117                 .caam = {
2118                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2119                                            OP_ALG_AAI_HMAC_PRECOMP,
2120                 },
2121         },
2122         {
2123                 .aead = {
2124                         .base = {
2125                                 .cra_name = "authenc(hmac(sha224),"
2126                                             "ecb(cipher_null))",
2127                                 .cra_driver_name = "authenc-hmac-sha224-"
2128                                                    "ecb-cipher_null-caam",
2129                                 .cra_blocksize = NULL_BLOCK_SIZE,
2130                         },
2131                         .setkey = aead_setkey,
2132                         .setauthsize = aead_setauthsize,
2133                         .encrypt = aead_encrypt,
2134                         .decrypt = aead_decrypt,
2135                         .ivsize = NULL_IV_SIZE,
2136                         .maxauthsize = SHA224_DIGEST_SIZE,
2137                 },
2138                 .caam = {
2139                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2140                                            OP_ALG_AAI_HMAC_PRECOMP,
2141                 },
2142         },
2143         {
2144                 .aead = {
2145                         .base = {
2146                                 .cra_name = "authenc(hmac(sha256),"
2147                                             "ecb(cipher_null))",
2148                                 .cra_driver_name = "authenc-hmac-sha256-"
2149                                                    "ecb-cipher_null-caam",
2150                                 .cra_blocksize = NULL_BLOCK_SIZE,
2151                         },
2152                         .setkey = aead_setkey,
2153                         .setauthsize = aead_setauthsize,
2154                         .encrypt = aead_encrypt,
2155                         .decrypt = aead_decrypt,
2156                         .ivsize = NULL_IV_SIZE,
2157                         .maxauthsize = SHA256_DIGEST_SIZE,
2158                 },
2159                 .caam = {
2160                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2161                                            OP_ALG_AAI_HMAC_PRECOMP,
2162                 },
2163         },
2164         {
2165                 .aead = {
2166                         .base = {
2167                                 .cra_name = "authenc(hmac(sha384),"
2168                                             "ecb(cipher_null))",
2169                                 .cra_driver_name = "authenc-hmac-sha384-"
2170                                                    "ecb-cipher_null-caam",
2171                                 .cra_blocksize = NULL_BLOCK_SIZE,
2172                         },
2173                         .setkey = aead_setkey,
2174                         .setauthsize = aead_setauthsize,
2175                         .encrypt = aead_encrypt,
2176                         .decrypt = aead_decrypt,
2177                         .ivsize = NULL_IV_SIZE,
2178                         .maxauthsize = SHA384_DIGEST_SIZE,
2179                 },
2180                 .caam = {
2181                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2182                                            OP_ALG_AAI_HMAC_PRECOMP,
2183                 },
2184         },
2185         {
2186                 .aead = {
2187                         .base = {
2188                                 .cra_name = "authenc(hmac(sha512),"
2189                                             "ecb(cipher_null))",
2190                                 .cra_driver_name = "authenc-hmac-sha512-"
2191                                                    "ecb-cipher_null-caam",
2192                                 .cra_blocksize = NULL_BLOCK_SIZE,
2193                         },
2194                         .setkey = aead_setkey,
2195                         .setauthsize = aead_setauthsize,
2196                         .encrypt = aead_encrypt,
2197                         .decrypt = aead_decrypt,
2198                         .ivsize = NULL_IV_SIZE,
2199                         .maxauthsize = SHA512_DIGEST_SIZE,
2200                 },
2201                 .caam = {
2202                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2203                                            OP_ALG_AAI_HMAC_PRECOMP,
2204                 },
2205         },
2206         {
2207                 .aead = {
2208                         .base = {
2209                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2210                                 .cra_driver_name = "authenc-hmac-md5-"
2211                                                    "cbc-aes-caam",
2212                                 .cra_blocksize = AES_BLOCK_SIZE,
2213                         },
2214                         .setkey = aead_setkey,
2215                         .setauthsize = aead_setauthsize,
2216                         .encrypt = aead_encrypt,
2217                         .decrypt = aead_decrypt,
2218                         .ivsize = AES_BLOCK_SIZE,
2219                         .maxauthsize = MD5_DIGEST_SIZE,
2220                 },
2221                 .caam = {
2222                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2223                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2224                                            OP_ALG_AAI_HMAC_PRECOMP,
2225                 },
2226         },
2227         {
2228                 .aead = {
2229                         .base = {
2230                                 .cra_name = "echainiv(authenc(hmac(md5),"
2231                                             "cbc(aes)))",
2232                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2233                                                    "cbc-aes-caam",
2234                                 .cra_blocksize = AES_BLOCK_SIZE,
2235                         },
2236                         .setkey = aead_setkey,
2237                         .setauthsize = aead_setauthsize,
2238                         .encrypt = aead_encrypt,
2239                         .decrypt = aead_decrypt,
2240                         .ivsize = AES_BLOCK_SIZE,
2241                         .maxauthsize = MD5_DIGEST_SIZE,
2242                 },
2243                 .caam = {
2244                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2245                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2246                                            OP_ALG_AAI_HMAC_PRECOMP,
2247                         .geniv = true,
2248                 },
2249         },
2250         {
2251                 .aead = {
2252                         .base = {
2253                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2254                                 .cra_driver_name = "authenc-hmac-sha1-"
2255                                                    "cbc-aes-caam",
2256                                 .cra_blocksize = AES_BLOCK_SIZE,
2257                         },
2258                         .setkey = aead_setkey,
2259                         .setauthsize = aead_setauthsize,
2260                         .encrypt = aead_encrypt,
2261                         .decrypt = aead_decrypt,
2262                         .ivsize = AES_BLOCK_SIZE,
2263                         .maxauthsize = SHA1_DIGEST_SIZE,
2264                 },
2265                 .caam = {
2266                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2267                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2268                                            OP_ALG_AAI_HMAC_PRECOMP,
2269                 },
2270         },
2271         {
2272                 .aead = {
2273                         .base = {
2274                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2275                                             "cbc(aes)))",
2276                                 .cra_driver_name = "echainiv-authenc-"
2277                                                    "hmac-sha1-cbc-aes-caam",
2278                                 .cra_blocksize = AES_BLOCK_SIZE,
2279                         },
2280                         .setkey = aead_setkey,
2281                         .setauthsize = aead_setauthsize,
2282                         .encrypt = aead_encrypt,
2283                         .decrypt = aead_decrypt,
2284                         .ivsize = AES_BLOCK_SIZE,
2285                         .maxauthsize = SHA1_DIGEST_SIZE,
2286                 },
2287                 .caam = {
2288                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2289                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2290                                            OP_ALG_AAI_HMAC_PRECOMP,
2291                         .geniv = true,
2292                 },
2293         },
2294         {
2295                 .aead = {
2296                         .base = {
2297                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2298                                 .cra_driver_name = "authenc-hmac-sha224-"
2299                                                    "cbc-aes-caam",
2300                                 .cra_blocksize = AES_BLOCK_SIZE,
2301                         },
2302                         .setkey = aead_setkey,
2303                         .setauthsize = aead_setauthsize,
2304                         .encrypt = aead_encrypt,
2305                         .decrypt = aead_decrypt,
2306                         .ivsize = AES_BLOCK_SIZE,
2307                         .maxauthsize = SHA224_DIGEST_SIZE,
2308                 },
2309                 .caam = {
2310                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2311                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2312                                            OP_ALG_AAI_HMAC_PRECOMP,
2313                 },
2314         },
2315         {
2316                 .aead = {
2317                         .base = {
2318                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2319                                             "cbc(aes)))",
2320                                 .cra_driver_name = "echainiv-authenc-"
2321                                                    "hmac-sha224-cbc-aes-caam",
2322                                 .cra_blocksize = AES_BLOCK_SIZE,
2323                         },
2324                         .setkey = aead_setkey,
2325                         .setauthsize = aead_setauthsize,
2326                         .encrypt = aead_encrypt,
2327                         .decrypt = aead_decrypt,
2328                         .ivsize = AES_BLOCK_SIZE,
2329                         .maxauthsize = SHA224_DIGEST_SIZE,
2330                 },
2331                 .caam = {
2332                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2333                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2334                                            OP_ALG_AAI_HMAC_PRECOMP,
2335                         .geniv = true,
2336                 },
2337         },
2338         {
2339                 .aead = {
2340                         .base = {
2341                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2342                                 .cra_driver_name = "authenc-hmac-sha256-"
2343                                                    "cbc-aes-caam",
2344                                 .cra_blocksize = AES_BLOCK_SIZE,
2345                         },
2346                         .setkey = aead_setkey,
2347                         .setauthsize = aead_setauthsize,
2348                         .encrypt = aead_encrypt,
2349                         .decrypt = aead_decrypt,
2350                         .ivsize = AES_BLOCK_SIZE,
2351                         .maxauthsize = SHA256_DIGEST_SIZE,
2352                 },
2353                 .caam = {
2354                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2355                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2356                                            OP_ALG_AAI_HMAC_PRECOMP,
2357                 },
2358         },
2359         {
2360                 .aead = {
2361                         .base = {
2362                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2363                                             "cbc(aes)))",
2364                                 .cra_driver_name = "echainiv-authenc-"
2365                                                    "hmac-sha256-cbc-aes-caam",
2366                                 .cra_blocksize = AES_BLOCK_SIZE,
2367                         },
2368                         .setkey = aead_setkey,
2369                         .setauthsize = aead_setauthsize,
2370                         .encrypt = aead_encrypt,
2371                         .decrypt = aead_decrypt,
2372                         .ivsize = AES_BLOCK_SIZE,
2373                         .maxauthsize = SHA256_DIGEST_SIZE,
2374                 },
2375                 .caam = {
2376                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2377                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2378                                            OP_ALG_AAI_HMAC_PRECOMP,
2379                         .geniv = true,
2380                 },
2381         },
2382         {
2383                 .aead = {
2384                         .base = {
2385                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2386                                 .cra_driver_name = "authenc-hmac-sha384-"
2387                                                    "cbc-aes-caam",
2388                                 .cra_blocksize = AES_BLOCK_SIZE,
2389                         },
2390                         .setkey = aead_setkey,
2391                         .setauthsize = aead_setauthsize,
2392                         .encrypt = aead_encrypt,
2393                         .decrypt = aead_decrypt,
2394                         .ivsize = AES_BLOCK_SIZE,
2395                         .maxauthsize = SHA384_DIGEST_SIZE,
2396                 },
2397                 .caam = {
2398                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2399                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2400                                            OP_ALG_AAI_HMAC_PRECOMP,
2401                 },
2402         },
2403         {
2404                 .aead = {
2405                         .base = {
2406                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2407                                             "cbc(aes)))",
2408                                 .cra_driver_name = "echainiv-authenc-"
2409                                                    "hmac-sha384-cbc-aes-caam",
2410                                 .cra_blocksize = AES_BLOCK_SIZE,
2411                         },
2412                         .setkey = aead_setkey,
2413                         .setauthsize = aead_setauthsize,
2414                         .encrypt = aead_encrypt,
2415                         .decrypt = aead_decrypt,
2416                         .ivsize = AES_BLOCK_SIZE,
2417                         .maxauthsize = SHA384_DIGEST_SIZE,
2418                 },
2419                 .caam = {
2420                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2421                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2422                                            OP_ALG_AAI_HMAC_PRECOMP,
2423                         .geniv = true,
2424                 },
2425         },
2426         {
2427                 .aead = {
2428                         .base = {
2429                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2430                                 .cra_driver_name = "authenc-hmac-sha512-"
2431                                                    "cbc-aes-caam",
2432                                 .cra_blocksize = AES_BLOCK_SIZE,
2433                         },
2434                         .setkey = aead_setkey,
2435                         .setauthsize = aead_setauthsize,
2436                         .encrypt = aead_encrypt,
2437                         .decrypt = aead_decrypt,
2438                         .ivsize = AES_BLOCK_SIZE,
2439                         .maxauthsize = SHA512_DIGEST_SIZE,
2440                 },
2441                 .caam = {
2442                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2443                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2444                                            OP_ALG_AAI_HMAC_PRECOMP,
2445                 },
2446         },
2447         {
2448                 .aead = {
2449                         .base = {
2450                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2451                                             "cbc(aes)))",
2452                                 .cra_driver_name = "echainiv-authenc-"
2453                                                    "hmac-sha512-cbc-aes-caam",
2454                                 .cra_blocksize = AES_BLOCK_SIZE,
2455                         },
2456                         .setkey = aead_setkey,
2457                         .setauthsize = aead_setauthsize,
2458                         .encrypt = aead_encrypt,
2459                         .decrypt = aead_decrypt,
2460                         .ivsize = AES_BLOCK_SIZE,
2461                         .maxauthsize = SHA512_DIGEST_SIZE,
2462                 },
2463                 .caam = {
2464                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2465                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2466                                            OP_ALG_AAI_HMAC_PRECOMP,
2467                         .geniv = true,
2468                 },
2469         },
2470         {
2471                 .aead = {
2472                         .base = {
2473                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2474                                 .cra_driver_name = "authenc-hmac-md5-"
2475                                                    "cbc-des3_ede-caam",
2476                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2477                         },
2478                         .setkey = des3_aead_setkey,
2479                         .setauthsize = aead_setauthsize,
2480                         .encrypt = aead_encrypt,
2481                         .decrypt = aead_decrypt,
2482                         .ivsize = DES3_EDE_BLOCK_SIZE,
2483                         .maxauthsize = MD5_DIGEST_SIZE,
2484                 },
2485                 .caam = {
2486                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2487                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2488                                            OP_ALG_AAI_HMAC_PRECOMP,
2489                 }
2490         },
2491         {
2492                 .aead = {
2493                         .base = {
2494                                 .cra_name = "echainiv(authenc(hmac(md5),"
2495                                             "cbc(des3_ede)))",
2496                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2497                                                    "cbc-des3_ede-caam",
2498                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2499                         },
2500                         .setkey = des3_aead_setkey,
2501                         .setauthsize = aead_setauthsize,
2502                         .encrypt = aead_encrypt,
2503                         .decrypt = aead_decrypt,
2504                         .ivsize = DES3_EDE_BLOCK_SIZE,
2505                         .maxauthsize = MD5_DIGEST_SIZE,
2506                 },
2507                 .caam = {
2508                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2509                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2510                                            OP_ALG_AAI_HMAC_PRECOMP,
2511                         .geniv = true,
2512                 }
2513         },
2514         {
2515                 .aead = {
2516                         .base = {
2517                                 .cra_name = "authenc(hmac(sha1),"
2518                                             "cbc(des3_ede))",
2519                                 .cra_driver_name = "authenc-hmac-sha1-"
2520                                                    "cbc-des3_ede-caam",
2521                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2522                         },
2523                         .setkey = des3_aead_setkey,
2524                         .setauthsize = aead_setauthsize,
2525                         .encrypt = aead_encrypt,
2526                         .decrypt = aead_decrypt,
2527                         .ivsize = DES3_EDE_BLOCK_SIZE,
2528                         .maxauthsize = SHA1_DIGEST_SIZE,
2529                 },
2530                 .caam = {
2531                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2532                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2533                                            OP_ALG_AAI_HMAC_PRECOMP,
2534                 },
2535         },
2536         {
2537                 .aead = {
2538                         .base = {
2539                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2540                                             "cbc(des3_ede)))",
2541                                 .cra_driver_name = "echainiv-authenc-"
2542                                                    "hmac-sha1-"
2543                                                    "cbc-des3_ede-caam",
2544                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2545                         },
2546                         .setkey = des3_aead_setkey,
2547                         .setauthsize = aead_setauthsize,
2548                         .encrypt = aead_encrypt,
2549                         .decrypt = aead_decrypt,
2550                         .ivsize = DES3_EDE_BLOCK_SIZE,
2551                         .maxauthsize = SHA1_DIGEST_SIZE,
2552                 },
2553                 .caam = {
2554                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2555                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2556                                            OP_ALG_AAI_HMAC_PRECOMP,
2557                         .geniv = true,
2558                 },
2559         },
2560         {
2561                 .aead = {
2562                         .base = {
2563                                 .cra_name = "authenc(hmac(sha224),"
2564                                             "cbc(des3_ede))",
2565                                 .cra_driver_name = "authenc-hmac-sha224-"
2566                                                    "cbc-des3_ede-caam",
2567                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2568                         },
2569                         .setkey = des3_aead_setkey,
2570                         .setauthsize = aead_setauthsize,
2571                         .encrypt = aead_encrypt,
2572                         .decrypt = aead_decrypt,
2573                         .ivsize = DES3_EDE_BLOCK_SIZE,
2574                         .maxauthsize = SHA224_DIGEST_SIZE,
2575                 },
2576                 .caam = {
2577                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2578                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2579                                            OP_ALG_AAI_HMAC_PRECOMP,
2580                 },
2581         },
2582         {
2583                 .aead = {
2584                         .base = {
2585                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2586                                             "cbc(des3_ede)))",
2587                                 .cra_driver_name = "echainiv-authenc-"
2588                                                    "hmac-sha224-"
2589                                                    "cbc-des3_ede-caam",
2590                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2591                         },
2592                         .setkey = des3_aead_setkey,
2593                         .setauthsize = aead_setauthsize,
2594                         .encrypt = aead_encrypt,
2595                         .decrypt = aead_decrypt,
2596                         .ivsize = DES3_EDE_BLOCK_SIZE,
2597                         .maxauthsize = SHA224_DIGEST_SIZE,
2598                 },
2599                 .caam = {
2600                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2601                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2602                                            OP_ALG_AAI_HMAC_PRECOMP,
2603                         .geniv = true,
2604                 },
2605         },
2606         {
2607                 .aead = {
2608                         .base = {
2609                                 .cra_name = "authenc(hmac(sha256),"
2610                                             "cbc(des3_ede))",
2611                                 .cra_driver_name = "authenc-hmac-sha256-"
2612                                                    "cbc-des3_ede-caam",
2613                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2614                         },
2615                         .setkey = des3_aead_setkey,
2616                         .setauthsize = aead_setauthsize,
2617                         .encrypt = aead_encrypt,
2618                         .decrypt = aead_decrypt,
2619                         .ivsize = DES3_EDE_BLOCK_SIZE,
2620                         .maxauthsize = SHA256_DIGEST_SIZE,
2621                 },
2622                 .caam = {
2623                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2624                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2625                                            OP_ALG_AAI_HMAC_PRECOMP,
2626                 },
2627         },
2628         {
2629                 .aead = {
2630                         .base = {
2631                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2632                                             "cbc(des3_ede)))",
2633                                 .cra_driver_name = "echainiv-authenc-"
2634                                                    "hmac-sha256-"
2635                                                    "cbc-des3_ede-caam",
2636                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2637                         },
2638                         .setkey = des3_aead_setkey,
2639                         .setauthsize = aead_setauthsize,
2640                         .encrypt = aead_encrypt,
2641                         .decrypt = aead_decrypt,
2642                         .ivsize = DES3_EDE_BLOCK_SIZE,
2643                         .maxauthsize = SHA256_DIGEST_SIZE,
2644                 },
2645                 .caam = {
2646                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2647                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2648                                            OP_ALG_AAI_HMAC_PRECOMP,
2649                         .geniv = true,
2650                 },
2651         },
2652         {
2653                 .aead = {
2654                         .base = {
2655                                 .cra_name = "authenc(hmac(sha384),"
2656                                             "cbc(des3_ede))",
2657                                 .cra_driver_name = "authenc-hmac-sha384-"
2658                                                    "cbc-des3_ede-caam",
2659                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2660                         },
2661                         .setkey = des3_aead_setkey,
2662                         .setauthsize = aead_setauthsize,
2663                         .encrypt = aead_encrypt,
2664                         .decrypt = aead_decrypt,
2665                         .ivsize = DES3_EDE_BLOCK_SIZE,
2666                         .maxauthsize = SHA384_DIGEST_SIZE,
2667                 },
2668                 .caam = {
2669                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2670                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2671                                            OP_ALG_AAI_HMAC_PRECOMP,
2672                 },
2673         },
2674         {
2675                 .aead = {
2676                         .base = {
2677                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2678                                             "cbc(des3_ede)))",
2679                                 .cra_driver_name = "echainiv-authenc-"
2680                                                    "hmac-sha384-"
2681                                                    "cbc-des3_ede-caam",
2682                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2683                         },
2684                         .setkey = des3_aead_setkey,
2685                         .setauthsize = aead_setauthsize,
2686                         .encrypt = aead_encrypt,
2687                         .decrypt = aead_decrypt,
2688                         .ivsize = DES3_EDE_BLOCK_SIZE,
2689                         .maxauthsize = SHA384_DIGEST_SIZE,
2690                 },
2691                 .caam = {
2692                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2693                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2694                                            OP_ALG_AAI_HMAC_PRECOMP,
2695                         .geniv = true,
2696                 },
2697         },
2698         {
2699                 .aead = {
2700                         .base = {
2701                                 .cra_name = "authenc(hmac(sha512),"
2702                                             "cbc(des3_ede))",
2703                                 .cra_driver_name = "authenc-hmac-sha512-"
2704                                                    "cbc-des3_ede-caam",
2705                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2706                         },
2707                         .setkey = des3_aead_setkey,
2708                         .setauthsize = aead_setauthsize,
2709                         .encrypt = aead_encrypt,
2710                         .decrypt = aead_decrypt,
2711                         .ivsize = DES3_EDE_BLOCK_SIZE,
2712                         .maxauthsize = SHA512_DIGEST_SIZE,
2713                 },
2714                 .caam = {
2715                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2716                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2717                                            OP_ALG_AAI_HMAC_PRECOMP,
2718                 },
2719         },
2720         {
2721                 .aead = {
2722                         .base = {
2723                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2724                                             "cbc(des3_ede)))",
2725                                 .cra_driver_name = "echainiv-authenc-"
2726                                                    "hmac-sha512-"
2727                                                    "cbc-des3_ede-caam",
2728                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2729                         },
2730                         .setkey = des3_aead_setkey,
2731                         .setauthsize = aead_setauthsize,
2732                         .encrypt = aead_encrypt,
2733                         .decrypt = aead_decrypt,
2734                         .ivsize = DES3_EDE_BLOCK_SIZE,
2735                         .maxauthsize = SHA512_DIGEST_SIZE,
2736                 },
2737                 .caam = {
2738                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2739                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2740                                            OP_ALG_AAI_HMAC_PRECOMP,
2741                         .geniv = true,
2742                 },
2743         },
2744         {
2745                 .aead = {
2746                         .base = {
2747                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2748                                 .cra_driver_name = "authenc-hmac-md5-"
2749                                                    "cbc-des-caam",
2750                                 .cra_blocksize = DES_BLOCK_SIZE,
2751                         },
2752                         .setkey = aead_setkey,
2753                         .setauthsize = aead_setauthsize,
2754                         .encrypt = aead_encrypt,
2755                         .decrypt = aead_decrypt,
2756                         .ivsize = DES_BLOCK_SIZE,
2757                         .maxauthsize = MD5_DIGEST_SIZE,
2758                 },
2759                 .caam = {
2760                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2761                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2762                                            OP_ALG_AAI_HMAC_PRECOMP,
2763                 },
2764         },
2765         {
2766                 .aead = {
2767                         .base = {
2768                                 .cra_name = "echainiv(authenc(hmac(md5),"
2769                                             "cbc(des)))",
2770                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2771                                                    "cbc-des-caam",
2772                                 .cra_blocksize = DES_BLOCK_SIZE,
2773                         },
2774                         .setkey = aead_setkey,
2775                         .setauthsize = aead_setauthsize,
2776                         .encrypt = aead_encrypt,
2777                         .decrypt = aead_decrypt,
2778                         .ivsize = DES_BLOCK_SIZE,
2779                         .maxauthsize = MD5_DIGEST_SIZE,
2780                 },
2781                 .caam = {
2782                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2783                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2784                                            OP_ALG_AAI_HMAC_PRECOMP,
2785                         .geniv = true,
2786                 },
2787         },
2788         {
2789                 .aead = {
2790                         .base = {
2791                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2792                                 .cra_driver_name = "authenc-hmac-sha1-"
2793                                                    "cbc-des-caam",
2794                                 .cra_blocksize = DES_BLOCK_SIZE,
2795                         },
2796                         .setkey = aead_setkey,
2797                         .setauthsize = aead_setauthsize,
2798                         .encrypt = aead_encrypt,
2799                         .decrypt = aead_decrypt,
2800                         .ivsize = DES_BLOCK_SIZE,
2801                         .maxauthsize = SHA1_DIGEST_SIZE,
2802                 },
2803                 .caam = {
2804                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2805                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2806                                            OP_ALG_AAI_HMAC_PRECOMP,
2807                 },
2808         },
2809         {
2810                 .aead = {
2811                         .base = {
2812                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2813                                             "cbc(des)))",
2814                                 .cra_driver_name = "echainiv-authenc-"
2815                                                    "hmac-sha1-cbc-des-caam",
2816                                 .cra_blocksize = DES_BLOCK_SIZE,
2817                         },
2818                         .setkey = aead_setkey,
2819                         .setauthsize = aead_setauthsize,
2820                         .encrypt = aead_encrypt,
2821                         .decrypt = aead_decrypt,
2822                         .ivsize = DES_BLOCK_SIZE,
2823                         .maxauthsize = SHA1_DIGEST_SIZE,
2824                 },
2825                 .caam = {
2826                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2827                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2828                                            OP_ALG_AAI_HMAC_PRECOMP,
2829                         .geniv = true,
2830                 },
2831         },
2832         {
2833                 .aead = {
2834                         .base = {
2835                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2836                                 .cra_driver_name = "authenc-hmac-sha224-"
2837                                                    "cbc-des-caam",
2838                                 .cra_blocksize = DES_BLOCK_SIZE,
2839                         },
2840                         .setkey = aead_setkey,
2841                         .setauthsize = aead_setauthsize,
2842                         .encrypt = aead_encrypt,
2843                         .decrypt = aead_decrypt,
2844                         .ivsize = DES_BLOCK_SIZE,
2845                         .maxauthsize = SHA224_DIGEST_SIZE,
2846                 },
2847                 .caam = {
2848                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2849                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2850                                            OP_ALG_AAI_HMAC_PRECOMP,
2851                 },
2852         },
2853         {
2854                 .aead = {
2855                         .base = {
2856                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2857                                             "cbc(des)))",
2858                                 .cra_driver_name = "echainiv-authenc-"
2859                                                    "hmac-sha224-cbc-des-caam",
2860                                 .cra_blocksize = DES_BLOCK_SIZE,
2861                         },
2862                         .setkey = aead_setkey,
2863                         .setauthsize = aead_setauthsize,
2864                         .encrypt = aead_encrypt,
2865                         .decrypt = aead_decrypt,
2866                         .ivsize = DES_BLOCK_SIZE,
2867                         .maxauthsize = SHA224_DIGEST_SIZE,
2868                 },
2869                 .caam = {
2870                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2871                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2872                                            OP_ALG_AAI_HMAC_PRECOMP,
2873                         .geniv = true,
2874                 },
2875         },
2876         {
2877                 .aead = {
2878                         .base = {
2879                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2880                                 .cra_driver_name = "authenc-hmac-sha256-"
2881                                                    "cbc-des-caam",
2882                                 .cra_blocksize = DES_BLOCK_SIZE,
2883                         },
2884                         .setkey = aead_setkey,
2885                         .setauthsize = aead_setauthsize,
2886                         .encrypt = aead_encrypt,
2887                         .decrypt = aead_decrypt,
2888                         .ivsize = DES_BLOCK_SIZE,
2889                         .maxauthsize = SHA256_DIGEST_SIZE,
2890                 },
2891                 .caam = {
2892                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2893                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2894                                            OP_ALG_AAI_HMAC_PRECOMP,
2895                 },
2896         },
2897         {
2898                 .aead = {
2899                         .base = {
2900                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2901                                             "cbc(des)))",
2902                                 .cra_driver_name = "echainiv-authenc-"
2903                                                    "hmac-sha256-cbc-des-caam",
2904                                 .cra_blocksize = DES_BLOCK_SIZE,
2905                         },
2906                         .setkey = aead_setkey,
2907                         .setauthsize = aead_setauthsize,
2908                         .encrypt = aead_encrypt,
2909                         .decrypt = aead_decrypt,
2910                         .ivsize = DES_BLOCK_SIZE,
2911                         .maxauthsize = SHA256_DIGEST_SIZE,
2912                 },
2913                 .caam = {
2914                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2915                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2916                                            OP_ALG_AAI_HMAC_PRECOMP,
2917                         .geniv = true,
2918                 },
2919         },
2920         {
2921                 .aead = {
2922                         .base = {
2923                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2924                                 .cra_driver_name = "authenc-hmac-sha384-"
2925                                                    "cbc-des-caam",
2926                                 .cra_blocksize = DES_BLOCK_SIZE,
2927                         },
2928                         .setkey = aead_setkey,
2929                         .setauthsize = aead_setauthsize,
2930                         .encrypt = aead_encrypt,
2931                         .decrypt = aead_decrypt,
2932                         .ivsize = DES_BLOCK_SIZE,
2933                         .maxauthsize = SHA384_DIGEST_SIZE,
2934                 },
2935                 .caam = {
2936                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2937                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2938                                            OP_ALG_AAI_HMAC_PRECOMP,
2939                 },
2940         },
2941         {
2942                 .aead = {
2943                         .base = {
2944                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2945                                             "cbc(des)))",
2946                                 .cra_driver_name = "echainiv-authenc-"
2947                                                    "hmac-sha384-cbc-des-caam",
2948                                 .cra_blocksize = DES_BLOCK_SIZE,
2949                         },
2950                         .setkey = aead_setkey,
2951                         .setauthsize = aead_setauthsize,
2952                         .encrypt = aead_encrypt,
2953                         .decrypt = aead_decrypt,
2954                         .ivsize = DES_BLOCK_SIZE,
2955                         .maxauthsize = SHA384_DIGEST_SIZE,
2956                 },
2957                 .caam = {
2958                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2959                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2960                                            OP_ALG_AAI_HMAC_PRECOMP,
2961                         .geniv = true,
2962                 },
2963         },
2964         {
2965                 .aead = {
2966                         .base = {
2967                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2968                                 .cra_driver_name = "authenc-hmac-sha512-"
2969                                                    "cbc-des-caam",
2970                                 .cra_blocksize = DES_BLOCK_SIZE,
2971                         },
2972                         .setkey = aead_setkey,
2973                         .setauthsize = aead_setauthsize,
2974                         .encrypt = aead_encrypt,
2975                         .decrypt = aead_decrypt,
2976                         .ivsize = DES_BLOCK_SIZE,
2977                         .maxauthsize = SHA512_DIGEST_SIZE,
2978                 },
2979                 .caam = {
2980                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2981                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2982                                            OP_ALG_AAI_HMAC_PRECOMP,
2983                 },
2984         },
2985         {
2986                 .aead = {
2987                         .base = {
2988                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2989                                             "cbc(des)))",
2990                                 .cra_driver_name = "echainiv-authenc-"
2991                                                    "hmac-sha512-cbc-des-caam",
2992                                 .cra_blocksize = DES_BLOCK_SIZE,
2993                         },
2994                         .setkey = aead_setkey,
2995                         .setauthsize = aead_setauthsize,
2996                         .encrypt = aead_encrypt,
2997                         .decrypt = aead_decrypt,
2998                         .ivsize = DES_BLOCK_SIZE,
2999                         .maxauthsize = SHA512_DIGEST_SIZE,
3000                 },
3001                 .caam = {
3002                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3003                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3004                                            OP_ALG_AAI_HMAC_PRECOMP,
3005                         .geniv = true,
3006                 },
3007         },
3008         {
3009                 .aead = {
3010                         .base = {
3011                                 .cra_name = "authenc(hmac(md5),"
3012                                             "rfc3686(ctr(aes)))",
3013                                 .cra_driver_name = "authenc-hmac-md5-"
3014                                                    "rfc3686-ctr-aes-caam",
3015                                 .cra_blocksize = 1,
3016                         },
3017                         .setkey = aead_setkey,
3018                         .setauthsize = aead_setauthsize,
3019                         .encrypt = aead_encrypt,
3020                         .decrypt = aead_decrypt,
3021                         .ivsize = CTR_RFC3686_IV_SIZE,
3022                         .maxauthsize = MD5_DIGEST_SIZE,
3023                 },
3024                 .caam = {
3025                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3026                                            OP_ALG_AAI_CTR_MOD128,
3027                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3028                                            OP_ALG_AAI_HMAC_PRECOMP,
3029                         .rfc3686 = true,
3030                 },
3031         },
3032         {
3033                 .aead = {
3034                         .base = {
3035                                 .cra_name = "seqiv(authenc("
3036                                             "hmac(md5),rfc3686(ctr(aes))))",
3037                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
3038                                                    "rfc3686-ctr-aes-caam",
3039                                 .cra_blocksize = 1,
3040                         },
3041                         .setkey = aead_setkey,
3042                         .setauthsize = aead_setauthsize,
3043                         .encrypt = aead_encrypt,
3044                         .decrypt = aead_decrypt,
3045                         .ivsize = CTR_RFC3686_IV_SIZE,
3046                         .maxauthsize = MD5_DIGEST_SIZE,
3047                 },
3048                 .caam = {
3049                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3050                                            OP_ALG_AAI_CTR_MOD128,
3051                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3052                                            OP_ALG_AAI_HMAC_PRECOMP,
3053                         .rfc3686 = true,
3054                         .geniv = true,
3055                 },
3056         },
3057         {
3058                 .aead = {
3059                         .base = {
3060                                 .cra_name = "authenc(hmac(sha1),"
3061                                             "rfc3686(ctr(aes)))",
3062                                 .cra_driver_name = "authenc-hmac-sha1-"
3063                                                    "rfc3686-ctr-aes-caam",
3064                                 .cra_blocksize = 1,
3065                         },
3066                         .setkey = aead_setkey,
3067                         .setauthsize = aead_setauthsize,
3068                         .encrypt = aead_encrypt,
3069                         .decrypt = aead_decrypt,
3070                         .ivsize = CTR_RFC3686_IV_SIZE,
3071                         .maxauthsize = SHA1_DIGEST_SIZE,
3072                 },
3073                 .caam = {
3074                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3075                                            OP_ALG_AAI_CTR_MOD128,
3076                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3077                                            OP_ALG_AAI_HMAC_PRECOMP,
3078                         .rfc3686 = true,
3079                 },
3080         },
3081         {
3082                 .aead = {
3083                         .base = {
3084                                 .cra_name = "seqiv(authenc("
3085                                             "hmac(sha1),rfc3686(ctr(aes))))",
3086                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3087                                                    "rfc3686-ctr-aes-caam",
3088                                 .cra_blocksize = 1,
3089                         },
3090                         .setkey = aead_setkey,
3091                         .setauthsize = aead_setauthsize,
3092                         .encrypt = aead_encrypt,
3093                         .decrypt = aead_decrypt,
3094                         .ivsize = CTR_RFC3686_IV_SIZE,
3095                         .maxauthsize = SHA1_DIGEST_SIZE,
3096                 },
3097                 .caam = {
3098                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3099                                            OP_ALG_AAI_CTR_MOD128,
3100                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3101                                            OP_ALG_AAI_HMAC_PRECOMP,
3102                         .rfc3686 = true,
3103                         .geniv = true,
3104                 },
3105         },
3106         {
3107                 .aead = {
3108                         .base = {
3109                                 .cra_name = "authenc(hmac(sha224),"
3110                                             "rfc3686(ctr(aes)))",
3111                                 .cra_driver_name = "authenc-hmac-sha224-"
3112                                                    "rfc3686-ctr-aes-caam",
3113                                 .cra_blocksize = 1,
3114                         },
3115                         .setkey = aead_setkey,
3116                         .setauthsize = aead_setauthsize,
3117                         .encrypt = aead_encrypt,
3118                         .decrypt = aead_decrypt,
3119                         .ivsize = CTR_RFC3686_IV_SIZE,
3120                         .maxauthsize = SHA224_DIGEST_SIZE,
3121                 },
3122                 .caam = {
3123                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3124                                            OP_ALG_AAI_CTR_MOD128,
3125                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3126                                            OP_ALG_AAI_HMAC_PRECOMP,
3127                         .rfc3686 = true,
3128                 },
3129         },
3130         {
3131                 .aead = {
3132                         .base = {
3133                                 .cra_name = "seqiv(authenc("
3134                                             "hmac(sha224),rfc3686(ctr(aes))))",
3135                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3136                                                    "rfc3686-ctr-aes-caam",
3137                                 .cra_blocksize = 1,
3138                         },
3139                         .setkey = aead_setkey,
3140                         .setauthsize = aead_setauthsize,
3141                         .encrypt = aead_encrypt,
3142                         .decrypt = aead_decrypt,
3143                         .ivsize = CTR_RFC3686_IV_SIZE,
3144                         .maxauthsize = SHA224_DIGEST_SIZE,
3145                 },
3146                 .caam = {
3147                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3148                                            OP_ALG_AAI_CTR_MOD128,
3149                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3150                                            OP_ALG_AAI_HMAC_PRECOMP,
3151                         .rfc3686 = true,
3152                         .geniv = true,
3153                 },
3154         },
3155         {
3156                 .aead = {
3157                         .base = {
3158                                 .cra_name = "authenc(hmac(sha256),"
3159                                             "rfc3686(ctr(aes)))",
3160                                 .cra_driver_name = "authenc-hmac-sha256-"
3161                                                    "rfc3686-ctr-aes-caam",
3162                                 .cra_blocksize = 1,
3163                         },
3164                         .setkey = aead_setkey,
3165                         .setauthsize = aead_setauthsize,
3166                         .encrypt = aead_encrypt,
3167                         .decrypt = aead_decrypt,
3168                         .ivsize = CTR_RFC3686_IV_SIZE,
3169                         .maxauthsize = SHA256_DIGEST_SIZE,
3170                 },
3171                 .caam = {
3172                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3173                                            OP_ALG_AAI_CTR_MOD128,
3174                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3175                                            OP_ALG_AAI_HMAC_PRECOMP,
3176                         .rfc3686 = true,
3177                 },
3178         },
3179         {
3180                 .aead = {
3181                         .base = {
3182                                 .cra_name = "seqiv(authenc(hmac(sha256),"
3183                                             "rfc3686(ctr(aes))))",
3184                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3185                                                    "rfc3686-ctr-aes-caam",
3186                                 .cra_blocksize = 1,
3187                         },
3188                         .setkey = aead_setkey,
3189                         .setauthsize = aead_setauthsize,
3190                         .encrypt = aead_encrypt,
3191                         .decrypt = aead_decrypt,
3192                         .ivsize = CTR_RFC3686_IV_SIZE,
3193                         .maxauthsize = SHA256_DIGEST_SIZE,
3194                 },
3195                 .caam = {
3196                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3197                                            OP_ALG_AAI_CTR_MOD128,
3198                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3199                                            OP_ALG_AAI_HMAC_PRECOMP,
3200                         .rfc3686 = true,
3201                         .geniv = true,
3202                 },
3203         },
3204         {
3205                 .aead = {
3206                         .base = {
3207                                 .cra_name = "authenc(hmac(sha384),"
3208                                             "rfc3686(ctr(aes)))",
3209                                 .cra_driver_name = "authenc-hmac-sha384-"
3210                                                    "rfc3686-ctr-aes-caam",
3211                                 .cra_blocksize = 1,
3212                         },
3213                         .setkey = aead_setkey,
3214                         .setauthsize = aead_setauthsize,
3215                         .encrypt = aead_encrypt,
3216                         .decrypt = aead_decrypt,
3217                         .ivsize = CTR_RFC3686_IV_SIZE,
3218                         .maxauthsize = SHA384_DIGEST_SIZE,
3219                 },
3220                 .caam = {
3221                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3222                                            OP_ALG_AAI_CTR_MOD128,
3223                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3224                                            OP_ALG_AAI_HMAC_PRECOMP,
3225                         .rfc3686 = true,
3226                 },
3227         },
3228         {
3229                 .aead = {
3230                         .base = {
3231                                 .cra_name = "seqiv(authenc(hmac(sha384),"
3232                                             "rfc3686(ctr(aes))))",
3233                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3234                                                    "rfc3686-ctr-aes-caam",
3235                                 .cra_blocksize = 1,
3236                         },
3237                         .setkey = aead_setkey,
3238                         .setauthsize = aead_setauthsize,
3239                         .encrypt = aead_encrypt,
3240                         .decrypt = aead_decrypt,
3241                         .ivsize = CTR_RFC3686_IV_SIZE,
3242                         .maxauthsize = SHA384_DIGEST_SIZE,
3243                 },
3244                 .caam = {
3245                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3246                                            OP_ALG_AAI_CTR_MOD128,
3247                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3248                                            OP_ALG_AAI_HMAC_PRECOMP,
3249                         .rfc3686 = true,
3250                         .geniv = true,
3251                 },
3252         },
3253         {
3254                 .aead = {
3255                         .base = {
3256                                 .cra_name = "authenc(hmac(sha512),"
3257                                             "rfc3686(ctr(aes)))",
3258                                 .cra_driver_name = "authenc-hmac-sha512-"
3259                                                    "rfc3686-ctr-aes-caam",
3260                                 .cra_blocksize = 1,
3261                         },
3262                         .setkey = aead_setkey,
3263                         .setauthsize = aead_setauthsize,
3264                         .encrypt = aead_encrypt,
3265                         .decrypt = aead_decrypt,
3266                         .ivsize = CTR_RFC3686_IV_SIZE,
3267                         .maxauthsize = SHA512_DIGEST_SIZE,
3268                 },
3269                 .caam = {
3270                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3271                                            OP_ALG_AAI_CTR_MOD128,
3272                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3273                                            OP_ALG_AAI_HMAC_PRECOMP,
3274                         .rfc3686 = true,
3275                 },
3276         },
3277         {
3278                 .aead = {
3279                         .base = {
3280                                 .cra_name = "seqiv(authenc(hmac(sha512),"
3281                                             "rfc3686(ctr(aes))))",
3282                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3283                                                    "rfc3686-ctr-aes-caam",
3284                                 .cra_blocksize = 1,
3285                         },
3286                         .setkey = aead_setkey,
3287                         .setauthsize = aead_setauthsize,
3288                         .encrypt = aead_encrypt,
3289                         .decrypt = aead_decrypt,
3290                         .ivsize = CTR_RFC3686_IV_SIZE,
3291                         .maxauthsize = SHA512_DIGEST_SIZE,
3292                 },
3293                 .caam = {
3294                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3295                                            OP_ALG_AAI_CTR_MOD128,
3296                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3297                                            OP_ALG_AAI_HMAC_PRECOMP,
3298                         .rfc3686 = true,
3299                         .geniv = true,
3300                 },
3301         },
3302         {
3303                 .aead = {
3304                         .base = {
3305                                 .cra_name = "rfc7539(chacha20,poly1305)",
3306                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
3307                                                    "caam",
3308                                 .cra_blocksize = 1,
3309                         },
3310                         .setkey = chachapoly_setkey,
3311                         .setauthsize = chachapoly_setauthsize,
3312                         .encrypt = chachapoly_encrypt,
3313                         .decrypt = chachapoly_decrypt,
3314                         .ivsize = CHACHAPOLY_IV_SIZE,
3315                         .maxauthsize = POLY1305_DIGEST_SIZE,
3316                 },
3317                 .caam = {
3318                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3319                                            OP_ALG_AAI_AEAD,
3320                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3321                                            OP_ALG_AAI_AEAD,
3322                         .nodkp = true,
3323                 },
3324         },
3325         {
3326                 .aead = {
3327                         .base = {
3328                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
3329                                 .cra_driver_name = "rfc7539esp-chacha20-"
3330                                                    "poly1305-caam",
3331                                 .cra_blocksize = 1,
3332                         },
3333                         .setkey = chachapoly_setkey,
3334                         .setauthsize = chachapoly_setauthsize,
3335                         .encrypt = chachapoly_encrypt,
3336                         .decrypt = chachapoly_decrypt,
3337                         .ivsize = 8,
3338                         .maxauthsize = POLY1305_DIGEST_SIZE,
3339                 },
3340                 .caam = {
3341                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3342                                            OP_ALG_AAI_AEAD,
3343                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3344                                            OP_ALG_AAI_AEAD,
3345                         .nodkp = true,
3346                 },
3347         },
3348 };
3349
3350 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3351                             bool uses_dkp)
3352 {
3353         dma_addr_t dma_addr;
3354         struct caam_drv_private *priv;
3355         const size_t sh_desc_enc_offset = offsetof(struct caam_ctx,
3356                                                    sh_desc_enc);
3357
3358         ctx->jrdev = caam_jr_alloc();
3359         if (IS_ERR(ctx->jrdev)) {
3360                 pr_err("Job Ring Device allocation for transform failed\n");
3361                 return PTR_ERR(ctx->jrdev);
3362         }
3363
3364         priv = dev_get_drvdata(ctx->jrdev->parent);
3365         if (priv->era >= 6 && uses_dkp)
3366                 ctx->dir = DMA_BIDIRECTIONAL;
3367         else
3368                 ctx->dir = DMA_TO_DEVICE;
3369
3370         dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3371                                         offsetof(struct caam_ctx,
3372                                                  sh_desc_enc_dma) -
3373                                         sh_desc_enc_offset,
3374                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3375         if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3376                 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3377                 caam_jr_free(ctx->jrdev);
3378                 return -ENOMEM;
3379         }
3380
3381         ctx->sh_desc_enc_dma = dma_addr;
3382         ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3383                                                    sh_desc_dec) -
3384                                         sh_desc_enc_offset;
3385         ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) -
3386                                         sh_desc_enc_offset;
3387
3388         /* copy descriptor header template value */
3389         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3390         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3391
3392         return 0;
3393 }
3394
3395 static int caam_cra_init(struct crypto_skcipher *tfm)
3396 {
3397         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3398         struct caam_skcipher_alg *caam_alg =
3399                 container_of(alg, typeof(*caam_alg), skcipher);
3400         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
3401         u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3402         int ret = 0;
3403
3404         ctx->enginectx.op.do_one_request = skcipher_do_one_req;
3405
3406         if (alg_aai == OP_ALG_AAI_XTS) {
3407                 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
3408                 struct crypto_skcipher *fallback;
3409
3410                 fallback = crypto_alloc_skcipher(tfm_name, 0,
3411                                                  CRYPTO_ALG_NEED_FALLBACK);
3412                 if (IS_ERR(fallback)) {
3413                         pr_err("Failed to allocate %s fallback: %ld\n",
3414                                tfm_name, PTR_ERR(fallback));
3415                         return PTR_ERR(fallback);
3416                 }
3417
3418                 ctx->fallback = fallback;
3419                 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
3420                                             crypto_skcipher_reqsize(fallback));
3421         } else {
3422                 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
3423         }
3424
3425         ret = caam_init_common(ctx, &caam_alg->caam, false);
3426         if (ret && ctx->fallback)
3427                 crypto_free_skcipher(ctx->fallback);
3428
3429         return ret;
3430 }
3431
3432 static int caam_aead_init(struct crypto_aead *tfm)
3433 {
3434         struct aead_alg *alg = crypto_aead_alg(tfm);
3435         struct caam_aead_alg *caam_alg =
3436                  container_of(alg, struct caam_aead_alg, aead);
3437         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
3438
3439         crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx));
3440
3441         ctx->enginectx.op.do_one_request = aead_do_one_req;
3442
3443         return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
3444 }
3445
3446 static void caam_exit_common(struct caam_ctx *ctx)
3447 {
3448         dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
3449                                offsetof(struct caam_ctx, sh_desc_enc_dma) -
3450                                offsetof(struct caam_ctx, sh_desc_enc),
3451                                ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3452         caam_jr_free(ctx->jrdev);
3453 }
3454
3455 static void caam_cra_exit(struct crypto_skcipher *tfm)
3456 {
3457         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
3458
3459         if (ctx->fallback)
3460                 crypto_free_skcipher(ctx->fallback);
3461         caam_exit_common(ctx);
3462 }
3463
3464 static void caam_aead_exit(struct crypto_aead *tfm)
3465 {
3466         caam_exit_common(crypto_aead_ctx(tfm));
3467 }
3468
3469 void caam_algapi_exit(void)
3470 {
3471         int i;
3472
3473         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3474                 struct caam_aead_alg *t_alg = driver_aeads + i;
3475
3476                 if (t_alg->registered)
3477                         crypto_unregister_aead(&t_alg->aead);
3478         }
3479
3480         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3481                 struct caam_skcipher_alg *t_alg = driver_algs + i;
3482
3483                 if (t_alg->registered)
3484                         crypto_unregister_skcipher(&t_alg->skcipher);
3485         }
3486 }
3487
3488 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3489 {
3490         struct skcipher_alg *alg = &t_alg->skcipher;
3491
3492         alg->base.cra_module = THIS_MODULE;
3493         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3494         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3495         alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3496                               CRYPTO_ALG_KERN_DRIVER_ONLY);
3497
3498         alg->init = caam_cra_init;
3499         alg->exit = caam_cra_exit;
3500 }
3501
3502 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3503 {
3504         struct aead_alg *alg = &t_alg->aead;
3505
3506         alg->base.cra_module = THIS_MODULE;
3507         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3508         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3509         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3510                               CRYPTO_ALG_KERN_DRIVER_ONLY;
3511
3512         alg->init = caam_aead_init;
3513         alg->exit = caam_aead_exit;
3514 }
3515
3516 int caam_algapi_init(struct device *ctrldev)
3517 {
3518         struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
3519         int i = 0, err = 0;
3520         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
3521         unsigned int md_limit = SHA512_DIGEST_SIZE;
3522         bool registered = false, gcm_support;
3523
3524         /*
3525          * Register crypto algorithms the device supports.
3526          * First, detect presence and attributes of DES, AES, and MD blocks.
3527          */
3528         if (priv->era < 10) {
3529                 u32 cha_vid, cha_inst, aes_rn;
3530
3531                 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
3532                 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
3533                 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3534
3535                 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
3536                 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
3537                            CHA_ID_LS_DES_SHIFT;
3538                 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
3539                 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3540                 ccha_inst = 0;
3541                 ptha_inst = 0;
3542
3543                 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) &
3544                          CHA_ID_LS_AES_MASK;
3545                 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
3546         } else {
3547                 u32 aesa, mdha;
3548
3549                 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
3550                 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
3551
3552                 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3553                 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3554
3555                 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
3556                 aes_inst = aesa & CHA_VER_NUM_MASK;
3557                 md_inst = mdha & CHA_VER_NUM_MASK;
3558                 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
3559                 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
3560
3561                 gcm_support = aesa & CHA_VER_MISC_AES_GCM;
3562         }
3563
3564         /* If MD is present, limit digest size based on LP256 */
3565         if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
3566                 md_limit = SHA256_DIGEST_SIZE;
3567
3568         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3569                 struct caam_skcipher_alg *t_alg = driver_algs + i;
3570                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
3571
3572                 /* Skip DES algorithms if not supported by device */
3573                 if (!des_inst &&
3574                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3575                      (alg_sel == OP_ALG_ALGSEL_DES)))
3576                                 continue;
3577
3578                 /* Skip AES algorithms if not supported by device */
3579                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3580                                 continue;
3581
3582                 /*
3583                  * Check support for AES modes not available
3584                  * on LP devices.
3585                  */
3586                 if (aes_vid == CHA_VER_VID_AES_LP &&
3587                     (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
3588                     OP_ALG_AAI_XTS)
3589                         continue;
3590
3591                 caam_skcipher_alg_init(t_alg);
3592
3593                 err = crypto_register_skcipher(&t_alg->skcipher);
3594                 if (err) {
3595                         pr_warn("%s alg registration failed\n",
3596                                 t_alg->skcipher.base.cra_driver_name);
3597                         continue;
3598                 }
3599
3600                 t_alg->registered = true;
3601                 registered = true;
3602         }
3603
3604         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3605                 struct caam_aead_alg *t_alg = driver_aeads + i;
3606                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3607                                  OP_ALG_ALGSEL_MASK;
3608                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3609                                  OP_ALG_ALGSEL_MASK;
3610                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3611
3612                 /* Skip DES algorithms if not supported by device */
3613                 if (!des_inst &&
3614                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3615                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3616                                 continue;
3617
3618                 /* Skip AES algorithms if not supported by device */
3619                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3620                                 continue;
3621
3622                 /* Skip CHACHA20 algorithms if not supported by device */
3623                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
3624                         continue;
3625
3626                 /* Skip POLY1305 algorithms if not supported by device */
3627                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
3628                         continue;
3629
3630                 /* Skip GCM algorithms if not supported by device */
3631                 if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
3632                     alg_aai == OP_ALG_AAI_GCM && !gcm_support)
3633                         continue;
3634
3635                 /*
3636                  * Skip algorithms requiring message digests
3637                  * if MD or MD size is not supported by device.
3638                  */
3639                 if (is_mdha(c2_alg_sel) &&
3640                     (!md_inst || t_alg->aead.maxauthsize > md_limit))
3641                         continue;
3642
3643                 caam_aead_alg_init(t_alg);
3644
3645                 err = crypto_register_aead(&t_alg->aead);
3646                 if (err) {
3647                         pr_warn("%s alg registration failed\n",
3648                                 t_alg->aead.base.cra_driver_name);
3649                         continue;
3650                 }
3651
3652                 t_alg->registered = true;
3653                 registered = true;
3654         }
3655
3656         if (registered)
3657                 pr_info("caam algorithms registered in /proc/crypto\n");
3658
3659         return err;
3660 }