drivers/crypto/caam/caamalg_desc.c

   1 /*
   2  * Shared descriptors for aead, ablkcipher algorithms
   3  *
   4  * Copyright 2016 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "desc_constr.h"
   9 #include "caamalg_desc.h"
  10
  11 /*
  12  * For aead functions, read payload and write payload,
  13  * both of which are specified in req->src and req->dst
  14  */
  15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
  16 {
  17         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
  18         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
  19                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
  20 }
  21
  22 /* Set DK bit in class 1 operation if shared */
  23 static inline void append_dec_op1(u32 *desc, u32 type)
  24 {
  25         u32 *jump_cmd, *uncond_jump_cmd;
  26
  27         /* DK bit is valid only for AES */
  28         if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
  29                 append_operation(desc, type | OP_ALG_AS_INITFINAL |
  30                                  OP_ALG_DECRYPT);
  31                 return;
  32         }
  33
  34         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
  35         append_operation(desc, type | OP_ALG_AS_INITFINAL |
  36                          OP_ALG_DECRYPT);
  37         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
  38         set_jump_tgt_here(desc, jump_cmd);
  39         append_operation(desc, type | OP_ALG_AS_INITFINAL |
  40                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
  41         set_jump_tgt_here(desc, uncond_jump_cmd);
  42 }
  43
  44 /**
  45  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
  46  *                               (non-protocol) with no (null) encryption.
  47  * @desc: pointer to buffer used for descriptor construction
  48  * @adata: pointer to authentication transform definitions.
  49  *         A split key is required for SEC Era < 6; the size of the split key
  50  *         is specified in this case. Valid algorithm values - one of
  51  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
  52  *         with OP_ALG_AAI_HMAC_PRECOMP.
  53  * @icvsize: integrity check value (ICV) size (truncated or full)
  54  * @era: SEC Era
  55  */
  56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
  57                                  unsigned int icvsize, int era)
  58 {
  59         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
  60
  61         init_sh_desc(desc, HDR_SHARE_SERIAL);
  62
  63         /* Skip if already shared */
  64         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
  65                                    JUMP_COND_SHRD);
  66         if (era < 6) {
  67                 if (adata->key_inline)
  68                         append_key_as_imm(desc, adata->key_virt,
  69                                           adata->keylen_pad, adata->keylen,
  70                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
  71                                           KEY_ENC);
  72                 else
  73                         append_key(desc, adata->key_dma, adata->keylen,
  74                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
  75         } else {
  76                 append_proto_dkp(desc, adata);
  77         }
  78         set_jump_tgt_here(desc, key_jump_cmd);
  79
  80         /* assoclen + cryptlen = seqinlen */
  81         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
  82
  83         /* Prepare to read and write cryptlen + assoclen bytes */
  84         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
  85         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
  86
  87         /*
  88          * MOVE_LEN opcode is not available in all SEC HW revisions,
  89          * thus need to do some magic, i.e. self-patch the descriptor
  90          * buffer.
  91          */
  92         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
  93                                     MOVE_DEST_MATH3 |
  94                                     (0x6 << MOVE_LEN_SHIFT));
  95         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
  96                                      MOVE_DEST_DESCBUF |
  97                                      MOVE_WAITCOMP |
  98                                      (0x8 << MOVE_LEN_SHIFT));
  99
 100         /* Class 2 operation */
 101         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 102                          OP_ALG_ENCRYPT);
 103
 104         /* Read and write cryptlen bytes */
 105         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 106
 107         set_move_tgt_here(desc, read_move_cmd);
 108         set_move_tgt_here(desc, write_move_cmd);
 109         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 110         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 111                     MOVE_AUX_LS);
 112
 113         /* Write ICV */
 114         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 115                          LDST_SRCDST_BYTE_CONTEXT);
 116
 117 #ifdef DEBUG
 118         print_hex_dump(KERN_ERR,
 119                        "aead null enc shdesc@" __stringify(__LINE__)": ",
 120                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 121 #endif
 122 }
 123 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
 124
 125 /**
 126  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
 127  *                               (non-protocol) with no (null) decryption.
 128  * @desc: pointer to buffer used for descriptor construction
 129  * @adata: pointer to authentication transform definitions.
 130  *         A split key is required for SEC Era < 6; the size of the split key
 131  *         is specified in this case. Valid algorithm values - one of
 132  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 133  *         with OP_ALG_AAI_HMAC_PRECOMP.
 134  * @icvsize: integrity check value (ICV) size (truncated or full)
 135  * @era: SEC Era
 136  */
 137 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
 138                                  unsigned int icvsize, int era)
 139 {
 140         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
 141
 142         init_sh_desc(desc, HDR_SHARE_SERIAL);
 143
 144         /* Skip if already shared */
 145         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 146                                    JUMP_COND_SHRD);
 147         if (era < 6) {
 148                 if (adata->key_inline)
 149                         append_key_as_imm(desc, adata->key_virt,
 150                                           adata->keylen_pad, adata->keylen,
 151                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
 152                                           KEY_ENC);
 153                 else
 154                         append_key(desc, adata->key_dma, adata->keylen,
 155                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 156         } else {
 157                 append_proto_dkp(desc, adata);
 158         }
 159         set_jump_tgt_here(desc, key_jump_cmd);
 160
 161         /* Class 2 operation */
 162         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 163                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 164
 165         /* assoclen + cryptlen = seqoutlen */
 166         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 167
 168         /* Prepare to read and write cryptlen + assoclen bytes */
 169         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 170         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 171
 172         /*
 173          * MOVE_LEN opcode is not available in all SEC HW revisions,
 174          * thus need to do some magic, i.e. self-patch the descriptor
 175          * buffer.
 176          */
 177         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 178                                     MOVE_DEST_MATH2 |
 179                                     (0x6 << MOVE_LEN_SHIFT));
 180         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 181                                      MOVE_DEST_DESCBUF |
 182                                      MOVE_WAITCOMP |
 183                                      (0x8 << MOVE_LEN_SHIFT));
 184
 185         /* Read and write cryptlen bytes */
 186         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 187
 188         /*
 189          * Insert a NOP here, since we need at least 4 instructions between
 190          * code patching the descriptor buffer and the location being patched.
 191          */
 192         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 193         set_jump_tgt_here(desc, jump_cmd);
 194
 195         set_move_tgt_here(desc, read_move_cmd);
 196         set_move_tgt_here(desc, write_move_cmd);
 197         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 198         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 199                     MOVE_AUX_LS);
 200         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 201
 202         /* Load ICV */
 203         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 204                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 205
 206 #ifdef DEBUG
 207         print_hex_dump(KERN_ERR,
 208                        "aead null dec shdesc@" __stringify(__LINE__)": ",
 209                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 210 #endif
 211 }
 212 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
 213
 214 static void init_sh_desc_key_aead(u32 * const desc,
 215                                   struct alginfo * const cdata,
 216                                   struct alginfo * const adata,
 217                                   const bool is_rfc3686, u32 *nonce, int era)
 218 {
 219         u32 *key_jump_cmd;
 220         unsigned int enckeylen = cdata->keylen;
 221
 222         /* Note: Context registers are saved. */
 223         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
 224
 225         /* Skip if already shared */
 226         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 227                                    JUMP_COND_SHRD);
 228
 229         /*
 230          * RFC3686 specific:
 231          *      | key = {AUTH_KEY, ENC_KEY, NONCE}
 232          *      | enckeylen = encryption key size + nonce size
 233          */
 234         if (is_rfc3686)
 235                 enckeylen -= CTR_RFC3686_NONCE_SIZE;
 236
 237         if (era < 6) {
 238                 if (adata->key_inline)
 239                         append_key_as_imm(desc, adata->key_virt,
 240                                           adata->keylen_pad, adata->keylen,
 241                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
 242                                           KEY_ENC);
 243                 else
 244                         append_key(desc, adata->key_dma, adata->keylen,
 245                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 246         } else {
 247                 append_proto_dkp(desc, adata);
 248         }
 249
 250         if (cdata->key_inline)
 251                 append_key_as_imm(desc, cdata->key_virt, enckeylen,
 252                                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 253         else
 254                 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
 255                            KEY_DEST_CLASS_REG);
 256
 257         /* Load Counter into CONTEXT1 reg */
 258         if (is_rfc3686) {
 259                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 260                                    LDST_CLASS_IND_CCB |
 261                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 262                 append_move(desc,
 263                             MOVE_SRC_OUTFIFO |
 264                             MOVE_DEST_CLASS1CTX |
 265                             (16 << MOVE_OFFSET_SHIFT) |
 266                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
 267         }
 268
 269         set_jump_tgt_here(desc, key_jump_cmd);
 270 }
 271
 272 /**
 273  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
 274  *                          (non-protocol).
 275  * @desc: pointer to buffer used for descriptor construction
 276  * @cdata: pointer to block cipher transform definitions
 277  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 278  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 279  * @adata: pointer to authentication transform definitions.
 280  *         A split key is required for SEC Era < 6; the size of the split key
 281  *         is specified in this case. Valid algorithm values - one of
 282  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 283  *         with OP_ALG_AAI_HMAC_PRECOMP.
 284  * @ivsize: initialization vector size
 285  * @icvsize: integrity check value (ICV) size (truncated or full)
 286  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 287  * @nonce: pointer to rfc3686 nonce
 288  * @ctx1_iv_off: IV offset in CONTEXT1 register
 289  * @is_qi: true when called from caam/qi
 290  * @era: SEC Era
 291  */
 292 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 293                             struct alginfo *adata, unsigned int ivsize,
 294                             unsigned int icvsize, const bool is_rfc3686,
 295                             u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
 296                             int era)
 297 {
 298         /* Note: Context registers are saved. */
 299         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 300
 301         /* Class 2 operation */
 302         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 303                          OP_ALG_ENCRYPT);
 304
 305         if (is_qi) {
 306                 u32 *wait_load_cmd;
 307
 308                 /* REG3 = assoclen */
 309                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 310                                 LDST_SRCDST_WORD_DECO_MATH3 |
 311                                 (4 << LDST_OFFSET_SHIFT));
 312
 313                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 314                                             JUMP_COND_CALM | JUMP_COND_NCP |
 315                                             JUMP_COND_NOP | JUMP_COND_NIP |
 316                                             JUMP_COND_NIFP);
 317                 set_jump_tgt_here(desc, wait_load_cmd);
 318
 319                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 320                                 LDST_SRCDST_BYTE_CONTEXT |
 321                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
 322         }
 323
 324         /* Read and write assoclen bytes */
 325         if (is_qi || era < 3) {
 326                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 327                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 328         } else {
 329                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 330                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 331         }
 332
 333         /* Skip assoc data */
 334         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 335
 336         /* read assoc before reading payload */
 337         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 338                                       FIFOLDST_VLF);
 339
 340         /* Load Counter into CONTEXT1 reg */
 341         if (is_rfc3686)
 342                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 343                                      LDST_SRCDST_BYTE_CONTEXT |
 344                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 345                                       LDST_OFFSET_SHIFT));
 346
 347         /* Class 1 operation */
 348         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 349                          OP_ALG_ENCRYPT);
 350
 351         /* Read and write cryptlen bytes */
 352         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 353         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 354         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 355
 356         /* Write ICV */
 357         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 358                          LDST_SRCDST_BYTE_CONTEXT);
 359
 360 #ifdef DEBUG
 361         print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
 362                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 363 #endif
 364 }
 365 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
 366
 367 /**
 368  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
 369  *                          (non-protocol).
 370  * @desc: pointer to buffer used for descriptor construction
 371  * @cdata: pointer to block cipher transform definitions
 372  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 373  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 374  * @adata: pointer to authentication transform definitions.
 375  *         A split key is required for SEC Era < 6; the size of the split key
 376  *         is specified in this case. Valid algorithm values - one of
 377  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 378  *         with OP_ALG_AAI_HMAC_PRECOMP.
 379  * @ivsize: initialization vector size
 380  * @icvsize: integrity check value (ICV) size (truncated or full)
 381  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 382  * @nonce: pointer to rfc3686 nonce
 383  * @ctx1_iv_off: IV offset in CONTEXT1 register
 384  * @is_qi: true when called from caam/qi
 385  * @era: SEC Era
 386  */
 387 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 388                             struct alginfo *adata, unsigned int ivsize,
 389                             unsigned int icvsize, const bool geniv,
 390                             const bool is_rfc3686, u32 *nonce,
 391                             const u32 ctx1_iv_off, const bool is_qi, int era)
 392 {
 393         /* Note: Context registers are saved. */
 394         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 395
 396         /* Class 2 operation */
 397         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 398                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 399
 400         if (is_qi) {
 401                 u32 *wait_load_cmd;
 402
 403                 /* REG3 = assoclen */
 404                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 405                                 LDST_SRCDST_WORD_DECO_MATH3 |
 406                                 (4 << LDST_OFFSET_SHIFT));
 407
 408                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 409                                             JUMP_COND_CALM | JUMP_COND_NCP |
 410                                             JUMP_COND_NOP | JUMP_COND_NIP |
 411                                             JUMP_COND_NIFP);
 412                 set_jump_tgt_here(desc, wait_load_cmd);
 413
 414                 if (!geniv)
 415                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 416                                         LDST_SRCDST_BYTE_CONTEXT |
 417                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
 418         }
 419
 420         /* Read and write assoclen bytes */
 421         if (is_qi || era < 3) {
 422                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 423                 if (geniv)
 424                         append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
 425                                                 ivsize);
 426                 else
 427                         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
 428                                         CAAM_CMD_SZ);
 429         } else {
 430                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 431                 if (geniv)
 432                         append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
 433                                                 ivsize);
 434                 else
 435                         append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
 436                                         CAAM_CMD_SZ);
 437         }
 438
 439         /* Skip assoc data */
 440         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 441
 442         /* read assoc before reading payload */
 443         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 444                              KEY_VLF);
 445
 446         if (geniv) {
 447                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 448                                 LDST_SRCDST_BYTE_CONTEXT |
 449                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
 450                 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
 451                             (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
 452         }
 453
 454         /* Load Counter into CONTEXT1 reg */
 455         if (is_rfc3686)
 456                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 457                                      LDST_SRCDST_BYTE_CONTEXT |
 458                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 459                                       LDST_OFFSET_SHIFT));
 460
 461         /* Choose operation */
 462         if (ctx1_iv_off)
 463                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 464                                  OP_ALG_DECRYPT);
 465         else
 466                 append_dec_op1(desc, cdata->algtype);
 467
 468         /* Read and write cryptlen bytes */
 469         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 470         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 471         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 472
 473         /* Load ICV */
 474         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 475                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 476
 477 #ifdef DEBUG
 478         print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
 479                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 480 #endif
 481 }
 482 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
 483
 484 /**
 485  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
 486  *                             (non-protocol) with HW-generated initialization
 487  *                             vector.
 488  * @desc: pointer to buffer used for descriptor construction
 489  * @cdata: pointer to block cipher transform definitions
 490  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 491  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 492  * @adata: pointer to authentication transform definitions.
 493  *         A split key is required for SEC Era < 6; the size of the split key
 494  *         is specified in this case. Valid algorithm values - one of
 495  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 496  *         with OP_ALG_AAI_HMAC_PRECOMP.
 497  * @ivsize: initialization vector size
 498  * @icvsize: integrity check value (ICV) size (truncated or full)
 499  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 500  * @nonce: pointer to rfc3686 nonce
 501  * @ctx1_iv_off: IV offset in CONTEXT1 register
 502  * @is_qi: true when called from caam/qi
 503  * @era: SEC Era
 504  */
 505 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 506                                struct alginfo *adata, unsigned int ivsize,
 507                                unsigned int icvsize, const bool is_rfc3686,
 508                                u32 *nonce, const u32 ctx1_iv_off,
 509                                const bool is_qi, int era)
 510 {
 511         u32 geniv, moveiv;
 512         u32 *wait_cmd;
 513
 514         /* Note: Context registers are saved. */
 515         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 516
 517         if (is_qi) {
 518                 u32 *wait_load_cmd;
 519
 520                 /* REG3 = assoclen */
 521                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 522                                 LDST_SRCDST_WORD_DECO_MATH3 |
 523                                 (4 << LDST_OFFSET_SHIFT));
 524
 525                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 526                                             JUMP_COND_CALM | JUMP_COND_NCP |
 527                                             JUMP_COND_NOP | JUMP_COND_NIP |
 528                                             JUMP_COND_NIFP);
 529                 set_jump_tgt_here(desc, wait_load_cmd);
 530         }
 531
 532         if (is_rfc3686) {
 533                 if (is_qi)
 534                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 535                                         LDST_SRCDST_BYTE_CONTEXT |
 536                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
 537
 538                 goto copy_iv;
 539         }
 540
 541         /* Generate IV */
 542         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 543                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 544                 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 545         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 546                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 547         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 548         append_move(desc, MOVE_WAITCOMP |
 549                     MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
 550                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 551                     (ivsize << MOVE_LEN_SHIFT));
 552         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 553
 554 copy_iv:
 555         /* Copy IV to class 1 context */
 556         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
 557                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 558                     (ivsize << MOVE_LEN_SHIFT));
 559
 560         /* Return to encryption */
 561         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 562                          OP_ALG_ENCRYPT);
 563
 564         /* Read and write assoclen bytes */
 565         if (is_qi || era < 3) {
 566                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 567                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 568         } else {
 569                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 570                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 571         }
 572
 573         /* Skip assoc data */
 574         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 575
 576         /* read assoc before reading payload */
 577         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 578                              KEY_VLF);
 579
 580         /* Copy iv from outfifo to class 2 fifo */
 581         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 582                  NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 583         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 584                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 585         append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
 586                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 587
 588         /* Load Counter into CONTEXT1 reg */
 589         if (is_rfc3686)
 590                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 591                                      LDST_SRCDST_BYTE_CONTEXT |
 592                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 593                                       LDST_OFFSET_SHIFT));
 594
 595         /* Class 1 operation */
 596         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 597                          OP_ALG_ENCRYPT);
 598
 599         /* Will write ivsize + cryptlen */
 600         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 601
 602         /* Not need to reload iv */
 603         append_seq_fifo_load(desc, ivsize,
 604                              FIFOLD_CLASS_SKIP);
 605
 606         /* Will read cryptlen */
 607         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 608
 609         /*
 610          * Wait for IV transfer (ofifo -> class2) to finish before starting
 611          * ciphertext transfer (ofifo -> external memory).
 612          */
 613         wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
 614         set_jump_tgt_here(desc, wait_cmd);
 615
 616         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
 617                              FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
 618         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 619
 620         /* Write ICV */
 621         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 622                          LDST_SRCDST_BYTE_CONTEXT);
 623
 624 #ifdef DEBUG
 625         print_hex_dump(KERN_ERR,
 626                        "aead givenc shdesc@" __stringify(__LINE__)": ",
 627                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 628 #endif
 629 }
 630 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
 631
 632 /**
 633  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
 634  * @desc: pointer to buffer used for descriptor construction
 635  * @cdata: pointer to block cipher transform definitions
 636  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 637  * @ivsize: initialization vector size
 638  * @icvsize: integrity check value (ICV) size (truncated or full)
 639  * @is_qi: true when called from caam/qi
 640  */
 641 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
 642                            unsigned int ivsize, unsigned int icvsize,
 643                            const bool is_qi)
 644 {
 645         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
 646             *zero_assoc_jump_cmd2;
 647
 648         init_sh_desc(desc, HDR_SHARE_SERIAL);
 649
 650         /* skip key loading if they are loaded due to sharing */
 651         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 652                                    JUMP_COND_SHRD);
 653         if (cdata->key_inline)
 654                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 655                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 656         else
 657                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 658                            KEY_DEST_CLASS_REG);
 659         set_jump_tgt_here(desc, key_jump_cmd);
 660
 661         /* class 1 operation */
 662         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 663                          OP_ALG_ENCRYPT);
 664
 665         if (is_qi) {
 666                 u32 *wait_load_cmd;
 667
 668                 /* REG3 = assoclen */
 669                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 670                                 LDST_SRCDST_WORD_DECO_MATH3 |
 671                                 (4 << LDST_OFFSET_SHIFT));
 672
 673                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 674                                             JUMP_COND_CALM | JUMP_COND_NCP |
 675                                             JUMP_COND_NOP | JUMP_COND_NIP |
 676                                             JUMP_COND_NIFP);
 677                 set_jump_tgt_here(desc, wait_load_cmd);
 678
 679                 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
 680                                         ivsize);
 681         } else {
 682                 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
 683                                 CAAM_CMD_SZ);
 684         }
 685
 686         /* if assoclen + cryptlen is ZERO, skip to ICV write */
 687         zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
 688                                                  JUMP_COND_MATH_Z);
 689
 690         if (is_qi)
 691                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 692                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 693
 694         /* if assoclen is ZERO, skip reading the assoc data */
 695         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 696         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 697                                            JUMP_COND_MATH_Z);
 698
 699         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 700
 701         /* skip assoc data */
 702         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 703
 704         /* cryptlen = seqinlen - assoclen */
 705         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 706
 707         /* if cryptlen is ZERO jump to zero-payload commands */
 708         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 709                                             JUMP_COND_MATH_Z);
 710
 711         /* read assoc data */
 712         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 713                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 714         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 715
 716         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 717
 718         /* write encrypted data */
 719         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 720
 721         /* read payload data */
 722         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 723                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 724
 725         /* jump to ICV writing */
 726         if (is_qi)
 727                 append_jump(desc, JUMP_TEST_ALL | 4);
 728         else
 729                 append_jump(desc, JUMP_TEST_ALL | 2);
 730
 731         /* zero-payload commands */
 732         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 733
 734         /* read assoc data */
 735         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 736                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 737         if (is_qi)
 738                 /* jump to ICV writing */
 739                 append_jump(desc, JUMP_TEST_ALL | 2);
 740
 741         /* There is no input data */
 742         set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
 743
 744         if (is_qi)
 745                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 746                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
 747                                      FIFOLD_TYPE_LAST1);
 748
 749         /* write ICV */
 750         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 751                          LDST_SRCDST_BYTE_CONTEXT);
 752
 753 #ifdef DEBUG
 754         print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
 755                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 756 #endif
 757 }
 758 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
 759
 760 /**
 761  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
 762  * @desc: pointer to buffer used for descriptor construction
 763  * @cdata: pointer to block cipher transform definitions
 764  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 765  * @ivsize: initialization vector size
 766  * @icvsize: integrity check value (ICV) size (truncated or full)
 767  * @is_qi: true when called from caam/qi
 768  */
 769 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
 770                            unsigned int ivsize, unsigned int icvsize,
 771                            const bool is_qi)
 772 {
 773         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
 774
 775         init_sh_desc(desc, HDR_SHARE_SERIAL);
 776
 777         /* skip key loading if they are loaded due to sharing */
 778         key_jump_cmd = append_jump(desc, JUMP_JSL |
 779                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
 780         if (cdata->key_inline)
 781                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 782                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 783         else
 784                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 785                            KEY_DEST_CLASS_REG);
 786         set_jump_tgt_here(desc, key_jump_cmd);
 787
 788         /* class 1 operation */
 789         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 790                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 791
 792         if (is_qi) {
 793                 u32 *wait_load_cmd;
 794
 795                 /* REG3 = assoclen */
 796                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 797                                 LDST_SRCDST_WORD_DECO_MATH3 |
 798                                 (4 << LDST_OFFSET_SHIFT));
 799
 800                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 801                                             JUMP_COND_CALM | JUMP_COND_NCP |
 802                                             JUMP_COND_NOP | JUMP_COND_NIP |
 803                                             JUMP_COND_NIFP);
 804                 set_jump_tgt_here(desc, wait_load_cmd);
 805
 806                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 807                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 808         }
 809
 810         /* if assoclen is ZERO, skip reading the assoc data */
 811         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 812         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 813                                                  JUMP_COND_MATH_Z);
 814
 815         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 816
 817         /* skip assoc data */
 818         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 819
 820         /* read assoc data */
 821         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 822                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 823
 824         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 825
 826         /* cryptlen = seqoutlen - assoclen */
 827         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 828
 829         /* jump to zero-payload command if cryptlen is zero */
 830         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 831                                             JUMP_COND_MATH_Z);
 832
 833         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 834
 835         /* store encrypted data */
 836         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 837
 838         /* read payload data */
 839         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 840                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 841
 842         /* zero-payload command */
 843         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 844
 845         /* read ICV */
 846         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
 847                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 848
 849 #ifdef DEBUG
 850         print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
 851                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 852 #endif
 853 }
 854 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
 855
 856 /**
 857  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
 858  *                             (non-protocol).
 859  * @desc: pointer to buffer used for descriptor construction
 860  * @cdata: pointer to block cipher transform definitions
 861  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 862  * @ivsize: initialization vector size
 863  * @icvsize: integrity check value (ICV) size (truncated or full)
 864  * @is_qi: true when called from caam/qi
 865  */
 866 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
 867                                unsigned int ivsize, unsigned int icvsize,
 868                                const bool is_qi)
 869 {
 870         u32 *key_jump_cmd;
 871
 872         init_sh_desc(desc, HDR_SHARE_SERIAL);
 873
 874         /* Skip key loading if it is loaded due to sharing */
 875         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 876                                    JUMP_COND_SHRD);
 877         if (cdata->key_inline)
 878                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 879                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 880         else
 881                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 882                            KEY_DEST_CLASS_REG);
 883         set_jump_tgt_here(desc, key_jump_cmd);
 884
 885         /* Class 1 operation */
 886         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 887                          OP_ALG_ENCRYPT);
 888
 889         if (is_qi) {
 890                 u32 *wait_load_cmd;
 891
 892                 /* REG3 = assoclen */
 893                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 894                                 LDST_SRCDST_WORD_DECO_MATH3 |
 895                                 (4 << LDST_OFFSET_SHIFT));
 896
 897                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 898                                             JUMP_COND_CALM | JUMP_COND_NCP |
 899                                             JUMP_COND_NOP | JUMP_COND_NIP |
 900                                             JUMP_COND_NIFP);
 901                 set_jump_tgt_here(desc, wait_load_cmd);
 902
 903                 /* Read salt and IV */
 904                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
 905                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
 906                                         FIFOLD_TYPE_IV);
 907                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 908                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 909         }
 910
 911         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
 912         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 913
 914         /* Read assoc data */
 915         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 916                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 917
 918         /* Skip IV */
 919         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
 920
 921         /* Will read cryptlen bytes */
 922         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 923
 924         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
 925         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
 926
 927         /* Skip assoc data */
 928         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 929
 930         /* cryptlen = seqoutlen - assoclen */
 931         append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
 932
 933         /* Write encrypted data */
 934         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 935
 936         /* Read payload data */
 937         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 938                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 939
 940         /* Write ICV */
 941         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 942                          LDST_SRCDST_BYTE_CONTEXT);
 943
 944 #ifdef DEBUG
 945         print_hex_dump(KERN_ERR,
 946                        "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
 947                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 948 #endif
 949 }
 950 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
 951
 952 /**
 953  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
 954  *                             (non-protocol).
 955  * @desc: pointer to buffer used for descriptor construction
 956  * @cdata: pointer to block cipher transform definitions
 957  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 958  * @ivsize: initialization vector size
 959  * @icvsize: integrity check value (ICV) size (truncated or full)
 960  * @is_qi: true when called from caam/qi
 961  */
 962 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
 963                                unsigned int ivsize, unsigned int icvsize,
 964                                const bool is_qi)
 965 {
 966         u32 *key_jump_cmd;
 967
 968         init_sh_desc(desc, HDR_SHARE_SERIAL);
 969
 970         /* Skip key loading if it is loaded due to sharing */
 971         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 972                                    JUMP_COND_SHRD);
 973         if (cdata->key_inline)
 974                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 975                                   cdata->keylen, CLASS_1 |
 976                                   KEY_DEST_CLASS_REG);
 977         else
 978                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 979                            KEY_DEST_CLASS_REG);
 980         set_jump_tgt_here(desc, key_jump_cmd);
 981
 982         /* Class 1 operation */
 983         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 984                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 985
 986         if (is_qi) {
 987                 u32 *wait_load_cmd;
 988
 989                 /* REG3 = assoclen */
 990                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 991                                 LDST_SRCDST_WORD_DECO_MATH3 |
 992                                 (4 << LDST_OFFSET_SHIFT));
 993
 994                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 995                                             JUMP_COND_CALM | JUMP_COND_NCP |
 996                                             JUMP_COND_NOP | JUMP_COND_NIP |
 997                                             JUMP_COND_NIFP);
 998                 set_jump_tgt_here(desc, wait_load_cmd);
 999
1000                 /* Read salt and IV */
1001                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1002                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1003                                         FIFOLD_TYPE_IV);
1004                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1005                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1006         }
1007
1008         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1009         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1010
1011         /* Read assoc data */
1012         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1013                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1014
1015         /* Skip IV */
1016         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1017
1018         /* Will read cryptlen bytes */
1019         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1020
1021         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1022         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1023
1024         /* Skip assoc data */
1025         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1026
1027         /* Will write cryptlen bytes */
1028         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1029
1030         /* Store payload data */
1031         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1032
1033         /* Read encrypted data */
1034         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1035                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1036
1037         /* Read ICV */
1038         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1039                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1040
1041 #ifdef DEBUG
1042         print_hex_dump(KERN_ERR,
1043                        "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1044                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1045 #endif
1046 }
1047 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1048
1049 /**
1050  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1051  *                             (non-protocol).
1052  * @desc: pointer to buffer used for descriptor construction
1053  * @cdata: pointer to block cipher transform definitions
1054  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1055  * @ivsize: initialization vector size
1056  * @icvsize: integrity check value (ICV) size (truncated or full)
1057  * @is_qi: true when called from caam/qi
1058  */
1059 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1060                                unsigned int ivsize, unsigned int icvsize,
1061                                const bool is_qi)
1062 {
1063         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1064
1065         init_sh_desc(desc, HDR_SHARE_SERIAL);
1066
1067         /* Skip key loading if it is loaded due to sharing */
1068         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1069                                    JUMP_COND_SHRD);
1070         if (cdata->key_inline)
1071                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1072                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1073         else
1074                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1075                            KEY_DEST_CLASS_REG);
1076         set_jump_tgt_here(desc, key_jump_cmd);
1077
1078         /* Class 1 operation */
1079         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1080                          OP_ALG_ENCRYPT);
1081
1082         if (is_qi) {
1083                 /* assoclen is not needed, skip it */
1084                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1085
1086                 /* Read salt and IV */
1087                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1088                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1089                                         FIFOLD_TYPE_IV);
1090                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1091                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1092         }
1093
1094         /* assoclen + cryptlen = seqinlen */
1095         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1096
1097         /*
1098          * MOVE_LEN opcode is not available in all SEC HW revisions,
1099          * thus need to do some magic, i.e. self-patch the descriptor
1100          * buffer.
1101          */
1102         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1103                                     (0x6 << MOVE_LEN_SHIFT));
1104         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1105                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1106
1107         /* Will read assoclen + cryptlen bytes */
1108         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1109
1110         /* Will write assoclen + cryptlen bytes */
1111         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1112
1113         /* Read and write assoclen + cryptlen bytes */
1114         aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1115
1116         set_move_tgt_here(desc, read_move_cmd);
1117         set_move_tgt_here(desc, write_move_cmd);
1118         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1119         /* Move payload data to OFIFO */
1120         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1121
1122         /* Write ICV */
1123         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1124                          LDST_SRCDST_BYTE_CONTEXT);
1125
1126 #ifdef DEBUG
1127         print_hex_dump(KERN_ERR,
1128                        "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1129                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1130 #endif
1131 }
1132 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1133
1134 /**
1135  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1136  *                             (non-protocol).
1137  * @desc: pointer to buffer used for descriptor construction
1138  * @cdata: pointer to block cipher transform definitions
1139  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1140  * @ivsize: initialization vector size
1141  * @icvsize: integrity check value (ICV) size (truncated or full)
1142  * @is_qi: true when called from caam/qi
1143  */
1144 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1145                                unsigned int ivsize, unsigned int icvsize,
1146                                const bool is_qi)
1147 {
1148         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1149
1150         init_sh_desc(desc, HDR_SHARE_SERIAL);
1151
1152         /* Skip key loading if it is loaded due to sharing */
1153         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1154                                    JUMP_COND_SHRD);
1155         if (cdata->key_inline)
1156                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1157                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1158         else
1159                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1160                            KEY_DEST_CLASS_REG);
1161         set_jump_tgt_here(desc, key_jump_cmd);
1162
1163         /* Class 1 operation */
1164         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1165                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1166
1167         if (is_qi) {
1168                 /* assoclen is not needed, skip it */
1169                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1170
1171                 /* Read salt and IV */
1172                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1173                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1174                                         FIFOLD_TYPE_IV);
1175                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1176                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1177         }
1178
1179         /* assoclen + cryptlen = seqoutlen */
1180         append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1181
1182         /*
1183          * MOVE_LEN opcode is not available in all SEC HW revisions,
1184          * thus need to do some magic, i.e. self-patch the descriptor
1185          * buffer.
1186          */
1187         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1188                                     (0x6 << MOVE_LEN_SHIFT));
1189         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1190                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1191
1192         /* Will read assoclen + cryptlen bytes */
1193         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1194
1195         /* Will write assoclen + cryptlen bytes */
1196         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1197
1198         /* Store payload data */
1199         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1200
1201         /* In-snoop assoclen + cryptlen data */
1202         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1203                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1204
1205         set_move_tgt_here(desc, read_move_cmd);
1206         set_move_tgt_here(desc, write_move_cmd);
1207         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1208         /* Move payload data to OFIFO */
1209         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1210         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1211
1212         /* Read ICV */
1213         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1214                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1215
1216 #ifdef DEBUG
1217         print_hex_dump(KERN_ERR,
1218                        "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1220 #endif
1221 }
1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223
1224 /*
1225  * For ablkcipher encrypt and decrypt, read from req->src and
1226  * write to req->dst
1227  */
1228 static inline void ablkcipher_append_src_dst(u32 *desc)
1229 {
1230         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1231         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1232         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1233                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1234         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1235 }
1236
1237 /**
1238  * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
1239  * @desc: pointer to buffer used for descriptor construction
1240  * @cdata: pointer to block cipher transform definitions
1241  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1242  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1243  * @ivsize: initialization vector size
1244  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1245  * @ctx1_iv_off: IV offset in CONTEXT1 register
1246  */
1247 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
1248                                   unsigned int ivsize, const bool is_rfc3686,
1249                                   const u32 ctx1_iv_off)
1250 {
1251         u32 *key_jump_cmd;
1252
1253         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1254         /* Skip if already shared */
1255         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1256                                    JUMP_COND_SHRD);
1257
1258         /* Load class1 key only */
1259         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1260                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1261
1262         /* Load nonce into CONTEXT1 reg */
1263         if (is_rfc3686) {
1264                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1265
1266                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1267                                    LDST_CLASS_IND_CCB |
1268                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1269                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1270                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1271                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1272         }
1273
1274         set_jump_tgt_here(desc, key_jump_cmd);
1275
1276         /* Load iv */
1277         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1278                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1279
1280         /* Load counter into CONTEXT1 reg */
1281         if (is_rfc3686)
1282                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1283                                      LDST_SRCDST_BYTE_CONTEXT |
1284                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1285                                       LDST_OFFSET_SHIFT));
1286
1287         /* Load operation */
1288         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1289                          OP_ALG_ENCRYPT);
1290
1291         /* Perform operation */
1292         ablkcipher_append_src_dst(desc);
1293
1294 #ifdef DEBUG
1295         print_hex_dump(KERN_ERR,
1296                        "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
1297                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1298 #endif
1299 }
1300 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
1301
1302 /**
1303  * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
1304  * @desc: pointer to buffer used for descriptor construction
1305  * @cdata: pointer to block cipher transform definitions
1306  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1307  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1308  * @ivsize: initialization vector size
1309  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1310  * @ctx1_iv_off: IV offset in CONTEXT1 register
1311  */
1312 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
1313                                   unsigned int ivsize, const bool is_rfc3686,
1314                                   const u32 ctx1_iv_off)
1315 {
1316         u32 *key_jump_cmd;
1317
1318         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1319         /* Skip if already shared */
1320         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1321                                    JUMP_COND_SHRD);
1322
1323         /* Load class1 key only */
1324         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1325                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1326
1327         /* Load nonce into CONTEXT1 reg */
1328         if (is_rfc3686) {
1329                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1330
1331                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1332                                    LDST_CLASS_IND_CCB |
1333                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1334                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1335                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1336                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1337         }
1338
1339         set_jump_tgt_here(desc, key_jump_cmd);
1340
1341         /* load IV */
1342         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1343                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1344
1345         /* Load counter into CONTEXT1 reg */
1346         if (is_rfc3686)
1347                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1348                                      LDST_SRCDST_BYTE_CONTEXT |
1349                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1350                                       LDST_OFFSET_SHIFT));
1351
1352         /* Choose operation */
1353         if (ctx1_iv_off)
1354                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1355                                  OP_ALG_DECRYPT);
1356         else
1357                 append_dec_op1(desc, cdata->algtype);
1358
1359         /* Perform operation */
1360         ablkcipher_append_src_dst(desc);
1361
1362 #ifdef DEBUG
1363         print_hex_dump(KERN_ERR,
1364                        "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
1365                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1366 #endif
1367 }
1368 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
1369
1370 /**
1371  * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
1372  *                                   with HW-generated initialization vector.
1373  * @desc: pointer to buffer used for descriptor construction
1374  * @cdata: pointer to block cipher transform definitions
1375  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1376  *         with OP_ALG_AAI_CBC.
1377  * @ivsize: initialization vector size
1378  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1379  * @ctx1_iv_off: IV offset in CONTEXT1 register
1380  */
1381 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
1382                                      unsigned int ivsize, const bool is_rfc3686,
1383                                      const u32 ctx1_iv_off)
1384 {
1385         u32 *key_jump_cmd, geniv;
1386
1387         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388         /* Skip if already shared */
1389         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390                                    JUMP_COND_SHRD);
1391
1392         /* Load class1 key only */
1393         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
1396         /* Load Nonce into CONTEXT1 reg */
1397         if (is_rfc3686) {
1398                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
1400                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401                                    LDST_CLASS_IND_CCB |
1402                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406         }
1407         set_jump_tgt_here(desc, key_jump_cmd);
1408
1409         /* Generate IV */
1410         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1411                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
1412                 (ivsize << NFIFOENTRY_DLEN_SHIFT);
1413         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1414                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1415         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1416         append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
1417                     MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
1418                     (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1419         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1420
1421         /* Copy generated IV to memory */
1422         append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1423                          LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1424
1425         /* Load Counter into CONTEXT1 reg */
1426         if (is_rfc3686)
1427                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1428                                      LDST_SRCDST_BYTE_CONTEXT |
1429                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1430                                       LDST_OFFSET_SHIFT));
1431
1432         if (ctx1_iv_off)
1433                 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1434                             (1 << JUMP_OFFSET_SHIFT));
1435
1436         /* Load operation */
1437         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1438                          OP_ALG_ENCRYPT);
1439
1440         /* Perform operation */
1441         ablkcipher_append_src_dst(desc);
1442
1443 #ifdef DEBUG
1444         print_hex_dump(KERN_ERR,
1445                        "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1446                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1447 #endif
1448 }
1449 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
1450
1451 /**
1452  * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
1453  *                                    descriptor
1454  * @desc: pointer to buffer used for descriptor construction
1455  * @cdata: pointer to block cipher transform definitions
1456  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1457  */
1458 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
1459 {
1460         /*
1461          * Set sector size to a big value, practically disabling
1462          * sector size segmentation in xts implementation. We cannot
1463          * take full advantage of this HW feature with existing
1464          * crypto API / dm-crypt SW architecture.
1465          */
1466         __be64 sector_size = cpu_to_be64(BIT(15));
1467         u32 *key_jump_cmd;
1468
1469         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1470         /* Skip if already shared */
1471         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1472                                    JUMP_COND_SHRD);
1473
1474         /* Load class1 keys only */
1475         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1476                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1477
1478         /* Load sector size with index 40 bytes (0x28) */
1479         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1480                            LDST_SRCDST_BYTE_CONTEXT |
1481                            (0x28 << LDST_OFFSET_SHIFT));
1482
1483         set_jump_tgt_here(desc, key_jump_cmd);
1484
1485         /*
1486          * create sequence for loading the sector index
1487          * Upper 8B of IV - will be used as sector index
1488          * Lower 8B of IV - will be discarded
1489          */
1490         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1491                         (0x20 << LDST_OFFSET_SHIFT));
1492         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1493
1494         /* Load operation */
1495         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1496                          OP_ALG_ENCRYPT);
1497
1498         /* Perform operation */
1499         ablkcipher_append_src_dst(desc);
1500
1501 #ifdef DEBUG
1502         print_hex_dump(KERN_ERR,
1503                        "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1504                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1505 #endif
1506 }
1507 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
1508
1509 /**
1510  * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
1511  *                                    descriptor
1512  * @desc: pointer to buffer used for descriptor construction
1513  * @cdata: pointer to block cipher transform definitions
1514  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1515  */
1516 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
1517 {
1518         /*
1519          * Set sector size to a big value, practically disabling
1520          * sector size segmentation in xts implementation. We cannot
1521          * take full advantage of this HW feature with existing
1522          * crypto API / dm-crypt SW architecture.
1523          */
1524         __be64 sector_size = cpu_to_be64(BIT(15));
1525         u32 *key_jump_cmd;
1526
1527         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1528         /* Skip if already shared */
1529         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1530                                    JUMP_COND_SHRD);
1531
1532         /* Load class1 key only */
1533         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1534                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1535
1536         /* Load sector size with index 40 bytes (0x28) */
1537         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1538                            LDST_SRCDST_BYTE_CONTEXT |
1539                            (0x28 << LDST_OFFSET_SHIFT));
1540
1541         set_jump_tgt_here(desc, key_jump_cmd);
1542
1543         /*
1544          * create sequence for loading the sector index
1545          * Upper 8B of IV - will be used as sector index
1546          * Lower 8B of IV - will be discarded
1547          */
1548         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1549                         (0x20 << LDST_OFFSET_SHIFT));
1550         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1551
1552         /* Load operation */
1553         append_dec_op1(desc, cdata->algtype);
1554
1555         /* Perform operation */
1556         ablkcipher_append_src_dst(desc);
1557
1558 #ifdef DEBUG
1559         print_hex_dump(KERN_ERR,
1560                        "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1561                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1562 #endif
1563 }
1564 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
1565
1566 MODULE_LICENSE("GPL");
1567 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1568 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");