GNU Linux-libre 4.19.314-gnu1
[releases.git] / drivers / crypto / caam / caamalg_desc.c
1 /*
2  * Shared descriptors for aead, ablkcipher algorithms
3  *
4  * Copyright 2016 NXP
5  */
6
7 #include "compat.h"
8 #include "desc_constr.h"
9 #include "caamalg_desc.h"
10
11 /*
12  * For aead functions, read payload and write payload,
13  * both of which are specified in req->src and req->dst
14  */
15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
16 {
17         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
18         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
19                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
20 }
21
22 /* Set DK bit in class 1 operation if shared */
23 static inline void append_dec_op1(u32 *desc, u32 type)
24 {
25         u32 *jump_cmd, *uncond_jump_cmd;
26
27         /* DK bit is valid only for AES */
28         if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
29                 append_operation(desc, type | OP_ALG_AS_INITFINAL |
30                                  OP_ALG_DECRYPT);
31                 return;
32         }
33
34         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
35         append_operation(desc, type | OP_ALG_AS_INITFINAL |
36                          OP_ALG_DECRYPT);
37         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38         set_jump_tgt_here(desc, jump_cmd);
39         append_operation(desc, type | OP_ALG_AS_INITFINAL |
40                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
41         set_jump_tgt_here(desc, uncond_jump_cmd);
42 }
43
44 /**
45  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46  *                               (non-protocol) with no (null) encryption.
47  * @desc: pointer to buffer used for descriptor construction
48  * @adata: pointer to authentication transform definitions.
49  *         A split key is required for SEC Era < 6; the size of the split key
50  *         is specified in this case. Valid algorithm values - one of
51  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
52  *         with OP_ALG_AAI_HMAC_PRECOMP.
53  * @icvsize: integrity check value (ICV) size (truncated or full)
54  * @era: SEC Era
55  */
56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57                                  unsigned int icvsize, int era)
58 {
59         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61         init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63         /* Skip if already shared */
64         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65                                    JUMP_COND_SHRD);
66         if (era < 6) {
67                 if (adata->key_inline)
68                         append_key_as_imm(desc, adata->key_virt,
69                                           adata->keylen_pad, adata->keylen,
70                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
71                                           KEY_ENC);
72                 else
73                         append_key(desc, adata->key_dma, adata->keylen,
74                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75         } else {
76                 append_proto_dkp(desc, adata);
77         }
78         set_jump_tgt_here(desc, key_jump_cmd);
79
80         /* assoclen + cryptlen = seqinlen */
81         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82
83         /* Prepare to read and write cryptlen + assoclen bytes */
84         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86
87         /*
88          * MOVE_LEN opcode is not available in all SEC HW revisions,
89          * thus need to do some magic, i.e. self-patch the descriptor
90          * buffer.
91          */
92         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93                                     MOVE_DEST_MATH3 |
94                                     (0x6 << MOVE_LEN_SHIFT));
95         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96                                      MOVE_DEST_DESCBUF |
97                                      MOVE_WAITCOMP |
98                                      (0x8 << MOVE_LEN_SHIFT));
99
100         /* Class 2 operation */
101         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
102                          OP_ALG_ENCRYPT);
103
104         /* Read and write cryptlen bytes */
105         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106
107         set_move_tgt_here(desc, read_move_cmd);
108         set_move_tgt_here(desc, write_move_cmd);
109         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111                     MOVE_AUX_LS);
112
113         /* Write ICV */
114         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115                          LDST_SRCDST_BYTE_CONTEXT);
116
117 #ifdef DEBUG
118         print_hex_dump(KERN_ERR,
119                        "aead null enc shdesc@" __stringify(__LINE__)": ",
120                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
121 #endif
122 }
123 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
124
125 /**
126  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
127  *                               (non-protocol) with no (null) decryption.
128  * @desc: pointer to buffer used for descriptor construction
129  * @adata: pointer to authentication transform definitions.
130  *         A split key is required for SEC Era < 6; the size of the split key
131  *         is specified in this case. Valid algorithm values - one of
132  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
133  *         with OP_ALG_AAI_HMAC_PRECOMP.
134  * @icvsize: integrity check value (ICV) size (truncated or full)
135  * @era: SEC Era
136  */
137 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
138                                  unsigned int icvsize, int era)
139 {
140         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
141
142         init_sh_desc(desc, HDR_SHARE_SERIAL);
143
144         /* Skip if already shared */
145         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
146                                    JUMP_COND_SHRD);
147         if (era < 6) {
148                 if (adata->key_inline)
149                         append_key_as_imm(desc, adata->key_virt,
150                                           adata->keylen_pad, adata->keylen,
151                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
152                                           KEY_ENC);
153                 else
154                         append_key(desc, adata->key_dma, adata->keylen,
155                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
156         } else {
157                 append_proto_dkp(desc, adata);
158         }
159         set_jump_tgt_here(desc, key_jump_cmd);
160
161         /* Class 2 operation */
162         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
163                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
164
165         /* assoclen + cryptlen = seqoutlen */
166         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
167
168         /* Prepare to read and write cryptlen + assoclen bytes */
169         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
170         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
171
172         /*
173          * MOVE_LEN opcode is not available in all SEC HW revisions,
174          * thus need to do some magic, i.e. self-patch the descriptor
175          * buffer.
176          */
177         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
178                                     MOVE_DEST_MATH2 |
179                                     (0x6 << MOVE_LEN_SHIFT));
180         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
181                                      MOVE_DEST_DESCBUF |
182                                      MOVE_WAITCOMP |
183                                      (0x8 << MOVE_LEN_SHIFT));
184
185         /* Read and write cryptlen bytes */
186         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
187
188         /*
189          * Insert a NOP here, since we need at least 4 instructions between
190          * code patching the descriptor buffer and the location being patched.
191          */
192         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
193         set_jump_tgt_here(desc, jump_cmd);
194
195         set_move_tgt_here(desc, read_move_cmd);
196         set_move_tgt_here(desc, write_move_cmd);
197         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
198         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
199                     MOVE_AUX_LS);
200         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
201
202         /* Load ICV */
203         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
204                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
205
206 #ifdef DEBUG
207         print_hex_dump(KERN_ERR,
208                        "aead null dec shdesc@" __stringify(__LINE__)": ",
209                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
210 #endif
211 }
212 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
213
214 static void init_sh_desc_key_aead(u32 * const desc,
215                                   struct alginfo * const cdata,
216                                   struct alginfo * const adata,
217                                   const bool is_rfc3686, u32 *nonce, int era)
218 {
219         u32 *key_jump_cmd;
220         unsigned int enckeylen = cdata->keylen;
221
222         /* Note: Context registers are saved. */
223         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
224
225         /* Skip if already shared */
226         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
227                                    JUMP_COND_SHRD);
228
229         /*
230          * RFC3686 specific:
231          *      | key = {AUTH_KEY, ENC_KEY, NONCE}
232          *      | enckeylen = encryption key size + nonce size
233          */
234         if (is_rfc3686)
235                 enckeylen -= CTR_RFC3686_NONCE_SIZE;
236
237         if (era < 6) {
238                 if (adata->key_inline)
239                         append_key_as_imm(desc, adata->key_virt,
240                                           adata->keylen_pad, adata->keylen,
241                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
242                                           KEY_ENC);
243                 else
244                         append_key(desc, adata->key_dma, adata->keylen,
245                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
246         } else {
247                 append_proto_dkp(desc, adata);
248         }
249
250         if (cdata->key_inline)
251                 append_key_as_imm(desc, cdata->key_virt, enckeylen,
252                                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
253         else
254                 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
255                            KEY_DEST_CLASS_REG);
256
257         /* Load Counter into CONTEXT1 reg */
258         if (is_rfc3686) {
259                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
260                                    LDST_CLASS_IND_CCB |
261                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
262                 append_move(desc,
263                             MOVE_SRC_OUTFIFO |
264                             MOVE_DEST_CLASS1CTX |
265                             (16 << MOVE_OFFSET_SHIFT) |
266                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
267         }
268
269         set_jump_tgt_here(desc, key_jump_cmd);
270 }
271
272 /**
273  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
274  *                          (non-protocol).
275  * @desc: pointer to buffer used for descriptor construction
276  * @cdata: pointer to block cipher transform definitions
277  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
278  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
279  * @adata: pointer to authentication transform definitions.
280  *         A split key is required for SEC Era < 6; the size of the split key
281  *         is specified in this case. Valid algorithm values - one of
282  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
283  *         with OP_ALG_AAI_HMAC_PRECOMP.
284  * @ivsize: initialization vector size
285  * @icvsize: integrity check value (ICV) size (truncated or full)
286  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
287  * @nonce: pointer to rfc3686 nonce
288  * @ctx1_iv_off: IV offset in CONTEXT1 register
289  * @is_qi: true when called from caam/qi
290  * @era: SEC Era
291  */
292 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
293                             struct alginfo *adata, unsigned int ivsize,
294                             unsigned int icvsize, const bool is_rfc3686,
295                             u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
296                             int era)
297 {
298         /* Note: Context registers are saved. */
299         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
300
301         /* Class 2 operation */
302         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
303                          OP_ALG_ENCRYPT);
304
305         if (is_qi) {
306                 u32 *wait_load_cmd;
307
308                 /* REG3 = assoclen */
309                 append_seq_load(desc, 4, LDST_CLASS_DECO |
310                                 LDST_SRCDST_WORD_DECO_MATH3 |
311                                 (4 << LDST_OFFSET_SHIFT));
312
313                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
314                                             JUMP_COND_CALM | JUMP_COND_NCP |
315                                             JUMP_COND_NOP | JUMP_COND_NIP |
316                                             JUMP_COND_NIFP);
317                 set_jump_tgt_here(desc, wait_load_cmd);
318
319                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
320                                 LDST_SRCDST_BYTE_CONTEXT |
321                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
322         }
323
324         /* Read and write assoclen bytes */
325         if (is_qi || era < 3) {
326                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
327                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
328         } else {
329                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
330                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
331         }
332
333         /* Skip assoc data */
334         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
335
336         /* read assoc before reading payload */
337         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
338                                       FIFOLDST_VLF);
339
340         /* Load Counter into CONTEXT1 reg */
341         if (is_rfc3686)
342                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
343                                      LDST_SRCDST_BYTE_CONTEXT |
344                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
345                                       LDST_OFFSET_SHIFT));
346
347         /* Class 1 operation */
348         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
349                          OP_ALG_ENCRYPT);
350
351         /* Read and write cryptlen bytes */
352         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
353         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
354         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
355
356         /* Write ICV */
357         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
358                          LDST_SRCDST_BYTE_CONTEXT);
359
360 #ifdef DEBUG
361         print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
362                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
363 #endif
364 }
365 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
366
367 /**
368  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
369  *                          (non-protocol).
370  * @desc: pointer to buffer used for descriptor construction
371  * @cdata: pointer to block cipher transform definitions
372  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
373  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
374  * @adata: pointer to authentication transform definitions.
375  *         A split key is required for SEC Era < 6; the size of the split key
376  *         is specified in this case. Valid algorithm values - one of
377  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
378  *         with OP_ALG_AAI_HMAC_PRECOMP.
379  * @ivsize: initialization vector size
380  * @icvsize: integrity check value (ICV) size (truncated or full)
381  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
382  * @nonce: pointer to rfc3686 nonce
383  * @ctx1_iv_off: IV offset in CONTEXT1 register
384  * @is_qi: true when called from caam/qi
385  * @era: SEC Era
386  */
387 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
388                             struct alginfo *adata, unsigned int ivsize,
389                             unsigned int icvsize, const bool geniv,
390                             const bool is_rfc3686, u32 *nonce,
391                             const u32 ctx1_iv_off, const bool is_qi, int era)
392 {
393         /* Note: Context registers are saved. */
394         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
395
396         /* Class 2 operation */
397         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
398                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
399
400         if (is_qi) {
401                 u32 *wait_load_cmd;
402
403                 /* REG3 = assoclen */
404                 append_seq_load(desc, 4, LDST_CLASS_DECO |
405                                 LDST_SRCDST_WORD_DECO_MATH3 |
406                                 (4 << LDST_OFFSET_SHIFT));
407
408                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
409                                             JUMP_COND_CALM | JUMP_COND_NCP |
410                                             JUMP_COND_NOP | JUMP_COND_NIP |
411                                             JUMP_COND_NIFP);
412                 set_jump_tgt_here(desc, wait_load_cmd);
413
414                 if (!geniv)
415                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
416                                         LDST_SRCDST_BYTE_CONTEXT |
417                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
418         }
419
420         /* Read and write assoclen bytes */
421         if (is_qi || era < 3) {
422                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
423                 if (geniv)
424                         append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
425                                                 ivsize);
426                 else
427                         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
428                                         CAAM_CMD_SZ);
429         } else {
430                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
431                 if (geniv)
432                         append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
433                                                 ivsize);
434                 else
435                         append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
436                                         CAAM_CMD_SZ);
437         }
438
439         /* Skip assoc data */
440         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
441
442         /* read assoc before reading payload */
443         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
444                              KEY_VLF);
445
446         if (geniv) {
447                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
448                                 LDST_SRCDST_BYTE_CONTEXT |
449                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
450                 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
451                             (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
452         }
453
454         /* Load Counter into CONTEXT1 reg */
455         if (is_rfc3686)
456                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
457                                      LDST_SRCDST_BYTE_CONTEXT |
458                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
459                                       LDST_OFFSET_SHIFT));
460
461         /* Choose operation */
462         if (ctx1_iv_off)
463                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
464                                  OP_ALG_DECRYPT);
465         else
466                 append_dec_op1(desc, cdata->algtype);
467
468         /* Read and write cryptlen bytes */
469         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
470         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
471         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
472
473         /* Load ICV */
474         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
475                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
476
477 #ifdef DEBUG
478         print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
479                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
480 #endif
481 }
482 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
483
484 /**
485  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
486  *                             (non-protocol) with HW-generated initialization
487  *                             vector.
488  * @desc: pointer to buffer used for descriptor construction
489  * @cdata: pointer to block cipher transform definitions
490  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
491  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
492  * @adata: pointer to authentication transform definitions.
493  *         A split key is required for SEC Era < 6; the size of the split key
494  *         is specified in this case. Valid algorithm values - one of
495  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
496  *         with OP_ALG_AAI_HMAC_PRECOMP.
497  * @ivsize: initialization vector size
498  * @icvsize: integrity check value (ICV) size (truncated or full)
499  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
500  * @nonce: pointer to rfc3686 nonce
501  * @ctx1_iv_off: IV offset in CONTEXT1 register
502  * @is_qi: true when called from caam/qi
503  * @era: SEC Era
504  */
505 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
506                                struct alginfo *adata, unsigned int ivsize,
507                                unsigned int icvsize, const bool is_rfc3686,
508                                u32 *nonce, const u32 ctx1_iv_off,
509                                const bool is_qi, int era)
510 {
511         u32 geniv, moveiv;
512         u32 *wait_cmd;
513
514         /* Note: Context registers are saved. */
515         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
516
517         if (is_qi) {
518                 u32 *wait_load_cmd;
519
520                 /* REG3 = assoclen */
521                 append_seq_load(desc, 4, LDST_CLASS_DECO |
522                                 LDST_SRCDST_WORD_DECO_MATH3 |
523                                 (4 << LDST_OFFSET_SHIFT));
524
525                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
526                                             JUMP_COND_CALM | JUMP_COND_NCP |
527                                             JUMP_COND_NOP | JUMP_COND_NIP |
528                                             JUMP_COND_NIFP);
529                 set_jump_tgt_here(desc, wait_load_cmd);
530         }
531
532         if (is_rfc3686) {
533                 if (is_qi)
534                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
535                                         LDST_SRCDST_BYTE_CONTEXT |
536                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
537
538                 goto copy_iv;
539         }
540
541         /* Generate IV */
542         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
543                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
544                 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
545         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
546                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
547         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
548         append_move(desc, MOVE_WAITCOMP |
549                     MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
550                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
551                     (ivsize << MOVE_LEN_SHIFT));
552         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
553
554 copy_iv:
555         /* Copy IV to class 1 context */
556         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
557                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
558                     (ivsize << MOVE_LEN_SHIFT));
559
560         /* Return to encryption */
561         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
562                          OP_ALG_ENCRYPT);
563
564         /* Read and write assoclen bytes */
565         if (is_qi || era < 3) {
566                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
567                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
568         } else {
569                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
570                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
571         }
572
573         /* Skip assoc data */
574         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
575
576         /* read assoc before reading payload */
577         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
578                              KEY_VLF);
579
580         /* Copy iv from outfifo to class 2 fifo */
581         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
582                  NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
583         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
584                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
585         append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
586                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
587
588         /* Load Counter into CONTEXT1 reg */
589         if (is_rfc3686)
590                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
591                                      LDST_SRCDST_BYTE_CONTEXT |
592                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
593                                       LDST_OFFSET_SHIFT));
594
595         /* Class 1 operation */
596         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
597                          OP_ALG_ENCRYPT);
598
599         /* Will write ivsize + cryptlen */
600         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
601
602         /* Not need to reload iv */
603         append_seq_fifo_load(desc, ivsize,
604                              FIFOLD_CLASS_SKIP);
605
606         /* Will read cryptlen */
607         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
608
609         /*
610          * Wait for IV transfer (ofifo -> class2) to finish before starting
611          * ciphertext transfer (ofifo -> external memory).
612          */
613         wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
614         set_jump_tgt_here(desc, wait_cmd);
615
616         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
617                              FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
618         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
619
620         /* Write ICV */
621         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
622                          LDST_SRCDST_BYTE_CONTEXT);
623
624 #ifdef DEBUG
625         print_hex_dump(KERN_ERR,
626                        "aead givenc shdesc@" __stringify(__LINE__)": ",
627                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
628 #endif
629 }
630 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
631
632 /**
633  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
634  * @desc: pointer to buffer used for descriptor construction
635  * @cdata: pointer to block cipher transform definitions
636  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
637  * @ivsize: initialization vector size
638  * @icvsize: integrity check value (ICV) size (truncated or full)
639  * @is_qi: true when called from caam/qi
640  */
641 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
642                            unsigned int ivsize, unsigned int icvsize,
643                            const bool is_qi)
644 {
645         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
646             *zero_assoc_jump_cmd2;
647
648         init_sh_desc(desc, HDR_SHARE_SERIAL);
649
650         /* skip key loading if they are loaded due to sharing */
651         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
652                                    JUMP_COND_SHRD);
653         if (cdata->key_inline)
654                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
655                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
656         else
657                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
658                            KEY_DEST_CLASS_REG);
659         set_jump_tgt_here(desc, key_jump_cmd);
660
661         /* class 1 operation */
662         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
663                          OP_ALG_ENCRYPT);
664
665         if (is_qi) {
666                 u32 *wait_load_cmd;
667
668                 /* REG3 = assoclen */
669                 append_seq_load(desc, 4, LDST_CLASS_DECO |
670                                 LDST_SRCDST_WORD_DECO_MATH3 |
671                                 (4 << LDST_OFFSET_SHIFT));
672
673                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
674                                             JUMP_COND_CALM | JUMP_COND_NCP |
675                                             JUMP_COND_NOP | JUMP_COND_NIP |
676                                             JUMP_COND_NIFP);
677                 set_jump_tgt_here(desc, wait_load_cmd);
678
679                 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
680                                         ivsize);
681         } else {
682                 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
683                                 CAAM_CMD_SZ);
684         }
685
686         /* if assoclen + cryptlen is ZERO, skip to ICV write */
687         zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
688                                                  JUMP_COND_MATH_Z);
689
690         if (is_qi)
691                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
692                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
693
694         /* if assoclen is ZERO, skip reading the assoc data */
695         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
696         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
697                                            JUMP_COND_MATH_Z);
698
699         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
700
701         /* skip assoc data */
702         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
703
704         /* cryptlen = seqinlen - assoclen */
705         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
706
707         /* if cryptlen is ZERO jump to zero-payload commands */
708         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
709                                             JUMP_COND_MATH_Z);
710
711         /* read assoc data */
712         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
713                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
714         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
715
716         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
717
718         /* write encrypted data */
719         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
720
721         /* read payload data */
722         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
723                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
724
725         /* jump to ICV writing */
726         if (is_qi)
727                 append_jump(desc, JUMP_TEST_ALL | 4);
728         else
729                 append_jump(desc, JUMP_TEST_ALL | 2);
730
731         /* zero-payload commands */
732         set_jump_tgt_here(desc, zero_payload_jump_cmd);
733
734         /* read assoc data */
735         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
736                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
737         if (is_qi)
738                 /* jump to ICV writing */
739                 append_jump(desc, JUMP_TEST_ALL | 2);
740
741         /* There is no input data */
742         set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
743
744         if (is_qi)
745                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
746                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
747                                      FIFOLD_TYPE_LAST1);
748
749         /* write ICV */
750         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
751                          LDST_SRCDST_BYTE_CONTEXT);
752
753 #ifdef DEBUG
754         print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
755                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
756 #endif
757 }
758 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
759
760 /**
761  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
762  * @desc: pointer to buffer used for descriptor construction
763  * @cdata: pointer to block cipher transform definitions
764  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
765  * @ivsize: initialization vector size
766  * @icvsize: integrity check value (ICV) size (truncated or full)
767  * @is_qi: true when called from caam/qi
768  */
769 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
770                            unsigned int ivsize, unsigned int icvsize,
771                            const bool is_qi)
772 {
773         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
774
775         init_sh_desc(desc, HDR_SHARE_SERIAL);
776
777         /* skip key loading if they are loaded due to sharing */
778         key_jump_cmd = append_jump(desc, JUMP_JSL |
779                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
780         if (cdata->key_inline)
781                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
782                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
783         else
784                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
785                            KEY_DEST_CLASS_REG);
786         set_jump_tgt_here(desc, key_jump_cmd);
787
788         /* class 1 operation */
789         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
790                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
791
792         if (is_qi) {
793                 u32 *wait_load_cmd;
794
795                 /* REG3 = assoclen */
796                 append_seq_load(desc, 4, LDST_CLASS_DECO |
797                                 LDST_SRCDST_WORD_DECO_MATH3 |
798                                 (4 << LDST_OFFSET_SHIFT));
799
800                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
801                                             JUMP_COND_CALM | JUMP_COND_NCP |
802                                             JUMP_COND_NOP | JUMP_COND_NIP |
803                                             JUMP_COND_NIFP);
804                 set_jump_tgt_here(desc, wait_load_cmd);
805
806                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
807                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
808         }
809
810         /* if assoclen is ZERO, skip reading the assoc data */
811         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
812         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
813                                                  JUMP_COND_MATH_Z);
814
815         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
816
817         /* skip assoc data */
818         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
819
820         /* read assoc data */
821         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
822                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
823
824         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
825
826         /* cryptlen = seqoutlen - assoclen */
827         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
828
829         /* jump to zero-payload command if cryptlen is zero */
830         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
831                                             JUMP_COND_MATH_Z);
832
833         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
834
835         /* store encrypted data */
836         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
837
838         /* read payload data */
839         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
840                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
841
842         /* zero-payload command */
843         set_jump_tgt_here(desc, zero_payload_jump_cmd);
844
845         /* read ICV */
846         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
847                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
848
849 #ifdef DEBUG
850         print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
851                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
852 #endif
853 }
854 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
855
856 /**
857  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
858  *                             (non-protocol).
859  * @desc: pointer to buffer used for descriptor construction
860  * @cdata: pointer to block cipher transform definitions
861  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
862  * @ivsize: initialization vector size
863  * @icvsize: integrity check value (ICV) size (truncated or full)
864  * @is_qi: true when called from caam/qi
865  */
866 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
867                                unsigned int ivsize, unsigned int icvsize,
868                                const bool is_qi)
869 {
870         u32 *key_jump_cmd;
871
872         init_sh_desc(desc, HDR_SHARE_SERIAL);
873
874         /* Skip key loading if it is loaded due to sharing */
875         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
876                                    JUMP_COND_SHRD);
877         if (cdata->key_inline)
878                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
879                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
880         else
881                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
882                            KEY_DEST_CLASS_REG);
883         set_jump_tgt_here(desc, key_jump_cmd);
884
885         /* Class 1 operation */
886         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
887                          OP_ALG_ENCRYPT);
888
889         if (is_qi) {
890                 u32 *wait_load_cmd;
891
892                 /* REG3 = assoclen */
893                 append_seq_load(desc, 4, LDST_CLASS_DECO |
894                                 LDST_SRCDST_WORD_DECO_MATH3 |
895                                 (4 << LDST_OFFSET_SHIFT));
896
897                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
898                                             JUMP_COND_CALM | JUMP_COND_NCP |
899                                             JUMP_COND_NOP | JUMP_COND_NIP |
900                                             JUMP_COND_NIFP);
901                 set_jump_tgt_here(desc, wait_load_cmd);
902
903                 /* Read salt and IV */
904                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
905                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
906                                         FIFOLD_TYPE_IV);
907                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
908                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
909         }
910
911         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
912         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
913
914         /* Read assoc data */
915         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
916                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
917
918         /* Skip IV */
919         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
920
921         /* Will read cryptlen bytes */
922         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
923
924         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
925         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
926
927         /* Skip assoc data */
928         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
929
930         /* cryptlen = seqoutlen - assoclen */
931         append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
932
933         /* Write encrypted data */
934         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
935
936         /* Read payload data */
937         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
938                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
939
940         /* Write ICV */
941         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
942                          LDST_SRCDST_BYTE_CONTEXT);
943
944 #ifdef DEBUG
945         print_hex_dump(KERN_ERR,
946                        "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
947                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
948 #endif
949 }
950 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
951
952 /**
953  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
954  *                             (non-protocol).
955  * @desc: pointer to buffer used for descriptor construction
956  * @cdata: pointer to block cipher transform definitions
957  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
958  * @ivsize: initialization vector size
959  * @icvsize: integrity check value (ICV) size (truncated or full)
960  * @is_qi: true when called from caam/qi
961  */
962 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
963                                unsigned int ivsize, unsigned int icvsize,
964                                const bool is_qi)
965 {
966         u32 *key_jump_cmd;
967
968         init_sh_desc(desc, HDR_SHARE_SERIAL);
969
970         /* Skip key loading if it is loaded due to sharing */
971         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
972                                    JUMP_COND_SHRD);
973         if (cdata->key_inline)
974                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
975                                   cdata->keylen, CLASS_1 |
976                                   KEY_DEST_CLASS_REG);
977         else
978                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
979                            KEY_DEST_CLASS_REG);
980         set_jump_tgt_here(desc, key_jump_cmd);
981
982         /* Class 1 operation */
983         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
984                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
985
986         if (is_qi) {
987                 u32 *wait_load_cmd;
988
989                 /* REG3 = assoclen */
990                 append_seq_load(desc, 4, LDST_CLASS_DECO |
991                                 LDST_SRCDST_WORD_DECO_MATH3 |
992                                 (4 << LDST_OFFSET_SHIFT));
993
994                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
995                                             JUMP_COND_CALM | JUMP_COND_NCP |
996                                             JUMP_COND_NOP | JUMP_COND_NIP |
997                                             JUMP_COND_NIFP);
998                 set_jump_tgt_here(desc, wait_load_cmd);
999
1000                 /* Read salt and IV */
1001                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1002                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1003                                         FIFOLD_TYPE_IV);
1004                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1005                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1006         }
1007
1008         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1009         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1010
1011         /* Read assoc data */
1012         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1013                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1014
1015         /* Skip IV */
1016         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1017
1018         /* Will read cryptlen bytes */
1019         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1020
1021         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1022         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1023
1024         /* Skip assoc data */
1025         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1026
1027         /* Will write cryptlen bytes */
1028         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1029
1030         /* Store payload data */
1031         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1032
1033         /* Read encrypted data */
1034         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1035                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1036
1037         /* Read ICV */
1038         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1039                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1040
1041 #ifdef DEBUG
1042         print_hex_dump(KERN_ERR,
1043                        "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1044                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1045 #endif
1046 }
1047 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1048
1049 /**
1050  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1051  *                             (non-protocol).
1052  * @desc: pointer to buffer used for descriptor construction
1053  * @cdata: pointer to block cipher transform definitions
1054  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1055  * @ivsize: initialization vector size
1056  * @icvsize: integrity check value (ICV) size (truncated or full)
1057  * @is_qi: true when called from caam/qi
1058  */
1059 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1060                                unsigned int ivsize, unsigned int icvsize,
1061                                const bool is_qi)
1062 {
1063         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1064
1065         init_sh_desc(desc, HDR_SHARE_SERIAL);
1066
1067         /* Skip key loading if it is loaded due to sharing */
1068         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1069                                    JUMP_COND_SHRD);
1070         if (cdata->key_inline)
1071                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1072                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1073         else
1074                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1075                            KEY_DEST_CLASS_REG);
1076         set_jump_tgt_here(desc, key_jump_cmd);
1077
1078         /* Class 1 operation */
1079         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1080                          OP_ALG_ENCRYPT);
1081
1082         if (is_qi) {
1083                 /* assoclen is not needed, skip it */
1084                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1085
1086                 /* Read salt and IV */
1087                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1088                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1089                                         FIFOLD_TYPE_IV);
1090                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1091                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1092         }
1093
1094         /* assoclen + cryptlen = seqinlen */
1095         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1096
1097         /*
1098          * MOVE_LEN opcode is not available in all SEC HW revisions,
1099          * thus need to do some magic, i.e. self-patch the descriptor
1100          * buffer.
1101          */
1102         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1103                                     (0x6 << MOVE_LEN_SHIFT));
1104         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1105                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1106
1107         /* Will read assoclen + cryptlen bytes */
1108         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1109
1110         /* Will write assoclen + cryptlen bytes */
1111         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1112
1113         /* Read and write assoclen + cryptlen bytes */
1114         aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1115
1116         set_move_tgt_here(desc, read_move_cmd);
1117         set_move_tgt_here(desc, write_move_cmd);
1118         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1119         /* Move payload data to OFIFO */
1120         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1121
1122         /* Write ICV */
1123         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1124                          LDST_SRCDST_BYTE_CONTEXT);
1125
1126 #ifdef DEBUG
1127         print_hex_dump(KERN_ERR,
1128                        "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1129                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1130 #endif
1131 }
1132 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1133
1134 /**
1135  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1136  *                             (non-protocol).
1137  * @desc: pointer to buffer used for descriptor construction
1138  * @cdata: pointer to block cipher transform definitions
1139  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1140  * @ivsize: initialization vector size
1141  * @icvsize: integrity check value (ICV) size (truncated or full)
1142  * @is_qi: true when called from caam/qi
1143  */
1144 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1145                                unsigned int ivsize, unsigned int icvsize,
1146                                const bool is_qi)
1147 {
1148         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1149
1150         init_sh_desc(desc, HDR_SHARE_SERIAL);
1151
1152         /* Skip key loading if it is loaded due to sharing */
1153         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1154                                    JUMP_COND_SHRD);
1155         if (cdata->key_inline)
1156                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1157                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1158         else
1159                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1160                            KEY_DEST_CLASS_REG);
1161         set_jump_tgt_here(desc, key_jump_cmd);
1162
1163         /* Class 1 operation */
1164         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1165                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1166
1167         if (is_qi) {
1168                 /* assoclen is not needed, skip it */
1169                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1170
1171                 /* Read salt and IV */
1172                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1173                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1174                                         FIFOLD_TYPE_IV);
1175                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1176                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1177         }
1178
1179         /* assoclen + cryptlen = seqoutlen */
1180         append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1181
1182         /*
1183          * MOVE_LEN opcode is not available in all SEC HW revisions,
1184          * thus need to do some magic, i.e. self-patch the descriptor
1185          * buffer.
1186          */
1187         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1188                                     (0x6 << MOVE_LEN_SHIFT));
1189         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1190                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1191
1192         /* Will read assoclen + cryptlen bytes */
1193         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1194
1195         /* Will write assoclen + cryptlen bytes */
1196         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1197
1198         /* Store payload data */
1199         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1200
1201         /* In-snoop assoclen + cryptlen data */
1202         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1203                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1204
1205         set_move_tgt_here(desc, read_move_cmd);
1206         set_move_tgt_here(desc, write_move_cmd);
1207         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1208         /* Move payload data to OFIFO */
1209         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1210         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1211
1212         /* Read ICV */
1213         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1214                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1215
1216 #ifdef DEBUG
1217         print_hex_dump(KERN_ERR,
1218                        "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1220 #endif
1221 }
1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223
1224 /*
1225  * For ablkcipher encrypt and decrypt, read from req->src and
1226  * write to req->dst
1227  */
1228 static inline void ablkcipher_append_src_dst(u32 *desc)
1229 {
1230         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1231         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1232         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1233                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1234         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1235 }
1236
1237 /**
1238  * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
1239  * @desc: pointer to buffer used for descriptor construction
1240  * @cdata: pointer to block cipher transform definitions
1241  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1242  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1243  * @ivsize: initialization vector size
1244  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1245  * @ctx1_iv_off: IV offset in CONTEXT1 register
1246  */
1247 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
1248                                   unsigned int ivsize, const bool is_rfc3686,
1249                                   const u32 ctx1_iv_off)
1250 {
1251         u32 *key_jump_cmd;
1252
1253         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1254         /* Skip if already shared */
1255         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1256                                    JUMP_COND_SHRD);
1257
1258         /* Load class1 key only */
1259         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1260                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1261
1262         /* Load nonce into CONTEXT1 reg */
1263         if (is_rfc3686) {
1264                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1265
1266                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1267                                    LDST_CLASS_IND_CCB |
1268                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1269                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1270                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1271                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1272         }
1273
1274         set_jump_tgt_here(desc, key_jump_cmd);
1275
1276         /* Load iv */
1277         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1278                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1279
1280         /* Load counter into CONTEXT1 reg */
1281         if (is_rfc3686)
1282                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1283                                      LDST_SRCDST_BYTE_CONTEXT |
1284                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1285                                       LDST_OFFSET_SHIFT));
1286
1287         /* Load operation */
1288         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1289                          OP_ALG_ENCRYPT);
1290
1291         /* Perform operation */
1292         ablkcipher_append_src_dst(desc);
1293
1294 #ifdef DEBUG
1295         print_hex_dump(KERN_ERR,
1296                        "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
1297                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1298 #endif
1299 }
1300 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
1301
1302 /**
1303  * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
1304  * @desc: pointer to buffer used for descriptor construction
1305  * @cdata: pointer to block cipher transform definitions
1306  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1307  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1308  * @ivsize: initialization vector size
1309  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1310  * @ctx1_iv_off: IV offset in CONTEXT1 register
1311  */
1312 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
1313                                   unsigned int ivsize, const bool is_rfc3686,
1314                                   const u32 ctx1_iv_off)
1315 {
1316         u32 *key_jump_cmd;
1317
1318         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1319         /* Skip if already shared */
1320         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1321                                    JUMP_COND_SHRD);
1322
1323         /* Load class1 key only */
1324         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1325                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1326
1327         /* Load nonce into CONTEXT1 reg */
1328         if (is_rfc3686) {
1329                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1330
1331                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1332                                    LDST_CLASS_IND_CCB |
1333                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1334                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1335                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1336                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1337         }
1338
1339         set_jump_tgt_here(desc, key_jump_cmd);
1340
1341         /* load IV */
1342         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1343                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1344
1345         /* Load counter into CONTEXT1 reg */
1346         if (is_rfc3686)
1347                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1348                                      LDST_SRCDST_BYTE_CONTEXT |
1349                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1350                                       LDST_OFFSET_SHIFT));
1351
1352         /* Choose operation */
1353         if (ctx1_iv_off)
1354                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1355                                  OP_ALG_DECRYPT);
1356         else
1357                 append_dec_op1(desc, cdata->algtype);
1358
1359         /* Perform operation */
1360         ablkcipher_append_src_dst(desc);
1361
1362 #ifdef DEBUG
1363         print_hex_dump(KERN_ERR,
1364                        "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
1365                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1366 #endif
1367 }
1368 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
1369
1370 /**
1371  * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
1372  *                                   with HW-generated initialization vector.
1373  * @desc: pointer to buffer used for descriptor construction
1374  * @cdata: pointer to block cipher transform definitions
1375  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1376  *         with OP_ALG_AAI_CBC.
1377  * @ivsize: initialization vector size
1378  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1379  * @ctx1_iv_off: IV offset in CONTEXT1 register
1380  */
1381 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
1382                                      unsigned int ivsize, const bool is_rfc3686,
1383                                      const u32 ctx1_iv_off)
1384 {
1385         u32 *key_jump_cmd, geniv;
1386
1387         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388         /* Skip if already shared */
1389         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390                                    JUMP_COND_SHRD);
1391
1392         /* Load class1 key only */
1393         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
1396         /* Load Nonce into CONTEXT1 reg */
1397         if (is_rfc3686) {
1398                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
1400                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401                                    LDST_CLASS_IND_CCB |
1402                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406         }
1407         set_jump_tgt_here(desc, key_jump_cmd);
1408
1409         /* Generate IV */
1410         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1411                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
1412                 (ivsize << NFIFOENTRY_DLEN_SHIFT);
1413         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1414                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1415         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1416         append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
1417                     MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
1418                     (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1419         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1420
1421         /* Copy generated IV to memory */
1422         append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1423                          LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1424
1425         /* Load Counter into CONTEXT1 reg */
1426         if (is_rfc3686)
1427                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1428                                      LDST_SRCDST_BYTE_CONTEXT |
1429                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1430                                       LDST_OFFSET_SHIFT));
1431
1432         if (ctx1_iv_off)
1433                 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1434                             (1 << JUMP_OFFSET_SHIFT));
1435
1436         /* Load operation */
1437         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1438                          OP_ALG_ENCRYPT);
1439
1440         /* Perform operation */
1441         ablkcipher_append_src_dst(desc);
1442
1443 #ifdef DEBUG
1444         print_hex_dump(KERN_ERR,
1445                        "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1446                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1447 #endif
1448 }
1449 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
1450
1451 /**
1452  * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
1453  *                                    descriptor
1454  * @desc: pointer to buffer used for descriptor construction
1455  * @cdata: pointer to block cipher transform definitions
1456  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1457  */
1458 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
1459 {
1460         /*
1461          * Set sector size to a big value, practically disabling
1462          * sector size segmentation in xts implementation. We cannot
1463          * take full advantage of this HW feature with existing
1464          * crypto API / dm-crypt SW architecture.
1465          */
1466         __be64 sector_size = cpu_to_be64(BIT(15));
1467         u32 *key_jump_cmd;
1468
1469         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1470         /* Skip if already shared */
1471         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1472                                    JUMP_COND_SHRD);
1473
1474         /* Load class1 keys only */
1475         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1476                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1477
1478         /* Load sector size with index 40 bytes (0x28) */
1479         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1480                            LDST_SRCDST_BYTE_CONTEXT |
1481                            (0x28 << LDST_OFFSET_SHIFT));
1482
1483         set_jump_tgt_here(desc, key_jump_cmd);
1484
1485         /*
1486          * create sequence for loading the sector index
1487          * Upper 8B of IV - will be used as sector index
1488          * Lower 8B of IV - will be discarded
1489          */
1490         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1491                         (0x20 << LDST_OFFSET_SHIFT));
1492         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1493
1494         /* Load operation */
1495         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1496                          OP_ALG_ENCRYPT);
1497
1498         /* Perform operation */
1499         ablkcipher_append_src_dst(desc);
1500
1501 #ifdef DEBUG
1502         print_hex_dump(KERN_ERR,
1503                        "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1504                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1505 #endif
1506 }
1507 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
1508
1509 /**
1510  * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
1511  *                                    descriptor
1512  * @desc: pointer to buffer used for descriptor construction
1513  * @cdata: pointer to block cipher transform definitions
1514  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1515  */
1516 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
1517 {
1518         /*
1519          * Set sector size to a big value, practically disabling
1520          * sector size segmentation in xts implementation. We cannot
1521          * take full advantage of this HW feature with existing
1522          * crypto API / dm-crypt SW architecture.
1523          */
1524         __be64 sector_size = cpu_to_be64(BIT(15));
1525         u32 *key_jump_cmd;
1526
1527         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1528         /* Skip if already shared */
1529         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1530                                    JUMP_COND_SHRD);
1531
1532         /* Load class1 key only */
1533         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1534                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1535
1536         /* Load sector size with index 40 bytes (0x28) */
1537         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1538                            LDST_SRCDST_BYTE_CONTEXT |
1539                            (0x28 << LDST_OFFSET_SHIFT));
1540
1541         set_jump_tgt_here(desc, key_jump_cmd);
1542
1543         /*
1544          * create sequence for loading the sector index
1545          * Upper 8B of IV - will be used as sector index
1546          * Lower 8B of IV - will be discarded
1547          */
1548         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1549                         (0x20 << LDST_OFFSET_SHIFT));
1550         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1551
1552         /* Load operation */
1553         append_dec_op1(desc, cdata->algtype);
1554
1555         /* Perform operation */
1556         ablkcipher_append_src_dst(desc);
1557
1558 #ifdef DEBUG
1559         print_hex_dump(KERN_ERR,
1560                        "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1561                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1562 #endif
1563 }
1564 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
1565
1566 MODULE_LICENSE("GPL");
1567 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1568 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");