drivers/crypto/caam/caamhash.c

   1 /*
   2  * caam - Freescale FSL CAAM support for ahash functions of crypto API
   3  *
   4  * Copyright 2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on caamalg.c crypto API driver.
   7  *
   8  * relationship of digest job descriptor or first job descriptor after init to
   9  * shared descriptors:
  10  *
  11  * ---------------                     ---------------
  12  * | JobDesc #1  |-------------------->|  ShareDesc  |
  13  * | *(packet 1) |                     |  (hashKey)  |
  14  * ---------------                     | (operation) |
  15  *                                     ---------------
  16  *
  17  * relationship of subsequent job descriptors to shared descriptors:
  18  *
  19  * ---------------                     ---------------
  20  * | JobDesc #2  |-------------------->|  ShareDesc  |
  21  * | *(packet 2) |      |------------->|  (hashKey)  |
  22  * ---------------      |    |-------->| (operation) |
  23  *       .              |    |         | (load ctx2) |
  24  *       .              |    |         ---------------
  25  * ---------------      |    |
  26  * | JobDesc #3  |------|    |
  27  * | *(packet 3) |           |
  28  * ---------------           |
  29  *       .                   |
  30  *       .                   |
  31  * ---------------           |
  32  * | JobDesc #4  |------------
  33  * | *(packet 4) |
  34  * ---------------
  35  *
  36  * The SharedDesc never changes for a connection unless rekeyed, but
  37  * each packet will likely be in a different place. So all we need
  38  * to know to process the packet is where the input is, where the
  39  * output goes, and what context we want to process with. Context is
  40  * in the SharedDesc, packet references in the JobDesc.
  41  *
  42  * So, a job desc looks like:
  43  *
  44  * ---------------------
  45  * | Header            |
  46  * | ShareDesc Pointer |
  47  * | SEQ_OUT_PTR       |
  48  * | (output buffer)   |
  49  * | (output length)   |
  50  * | SEQ_IN_PTR        |
  51  * | (input buffer)    |
  52  * | (input length)    |
  53  * ---------------------
  54  */
  55
  56 #include "compat.h"
  57
  58 #include "regs.h"
  59 #include "intern.h"
  60 #include "desc_constr.h"
  61 #include "jr.h"
  62 #include "error.h"
  63 #include "sg_sw_sec4.h"
  64 #include "key_gen.h"
  65
  66 #define CAAM_CRA_PRIORITY               3000
  67
  68 /* max hash key is max split key size */
  69 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
  70
  71 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
  72 #define CAAM_MAX_HASH_DIGEST_SIZE       SHA512_DIGEST_SIZE
  73
  74 /* length of descriptors text */
  75 #define DESC_AHASH_BASE                 (3 * CAAM_CMD_SZ)
  76 #define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
  77 #define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  78 #define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  79 #define DESC_AHASH_FINUP_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  80 #define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  81
  82 #define DESC_HASH_MAX_USED_BYTES        (DESC_AHASH_FINAL_LEN + \
  83                                          CAAM_MAX_HASH_KEY_SIZE)
  84 #define DESC_HASH_MAX_USED_LEN          (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
  85
  86 /* caam context sizes for hashes: running digest + 8 */
  87 #define HASH_MSG_LEN                    8
  88 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
  89
  90 #ifdef DEBUG
  91 /* for print_hex_dumps with line references */
  92 #define debug(format, arg...) printk(format, arg)
  93 #else
  94 #define debug(format, arg...)
  95 #endif
  96
  97
  98 static struct list_head hash_list;
  99
 100 /* ahash per-session context */
 101 struct caam_hash_ctx {
 102         u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 103         u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 104         u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 105         u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 106         dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 107         dma_addr_t sh_desc_update_first_dma;
 108         dma_addr_t sh_desc_fin_dma;
 109         dma_addr_t sh_desc_digest_dma;
 110         struct device *jrdev;
 111         u8 key[CAAM_MAX_HASH_KEY_SIZE];
 112         int ctx_len;
 113         struct alginfo adata;
 114 };
 115
 116 /* ahash state */
 117 struct caam_hash_state {
 118         dma_addr_t buf_dma;
 119         dma_addr_t ctx_dma;
 120         u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 121         int buflen_0;
 122         u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 123         int buflen_1;
 124         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
 125         int (*update)(struct ahash_request *req);
 126         int (*final)(struct ahash_request *req);
 127         int (*finup)(struct ahash_request *req);
 128         int current_buf;
 129 };
 130
 131 struct caam_export_state {
 132         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
 133         u8 caam_ctx[MAX_CTX_LEN];
 134         int buflen;
 135         int (*update)(struct ahash_request *req);
 136         int (*final)(struct ahash_request *req);
 137         int (*finup)(struct ahash_request *req);
 138 };
 139
 140 static inline void switch_buf(struct caam_hash_state *state)
 141 {
 142         state->current_buf ^= 1;
 143 }
 144
 145 static inline u8 *current_buf(struct caam_hash_state *state)
 146 {
 147         return state->current_buf ? state->buf_1 : state->buf_0;
 148 }
 149
 150 static inline u8 *alt_buf(struct caam_hash_state *state)
 151 {
 152         return state->current_buf ? state->buf_0 : state->buf_1;
 153 }
 154
 155 static inline int *current_buflen(struct caam_hash_state *state)
 156 {
 157         return state->current_buf ? &state->buflen_1 : &state->buflen_0;
 158 }
 159
 160 static inline int *alt_buflen(struct caam_hash_state *state)
 161 {
 162         return state->current_buf ? &state->buflen_0 : &state->buflen_1;
 163 }
 164
 165 /* Common job descriptor seq in/out ptr routines */
 166
 167 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
 168 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
 169                                       struct caam_hash_state *state,
 170                                       int ctx_len)
 171 {
 172         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
 173                                         ctx_len, DMA_FROM_DEVICE);
 174         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 175                 dev_err(jrdev, "unable to map ctx\n");
 176                 state->ctx_dma = 0;
 177                 return -ENOMEM;
 178         }
 179
 180         append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
 181
 182         return 0;
 183 }
 184
 185 /* Map req->result, and append seq_out_ptr command that points to it */
 186 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
 187                                                 u8 *result, int digestsize)
 188 {
 189         dma_addr_t dst_dma;
 190
 191         dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
 192         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 193
 194         return dst_dma;
 195 }
 196
 197 /* Map current buffer in state (if length > 0) and put it in link table */
 198 static inline int buf_map_to_sec4_sg(struct device *jrdev,
 199                                      struct sec4_sg_entry *sec4_sg,
 200                                      struct caam_hash_state *state)
 201 {
 202         int buflen = *current_buflen(state);
 203
 204         if (!buflen)
 205                 return 0;
 206
 207         state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
 208                                         DMA_TO_DEVICE);
 209         if (dma_mapping_error(jrdev, state->buf_dma)) {
 210                 dev_err(jrdev, "unable to map buf\n");
 211                 state->buf_dma = 0;
 212                 return -ENOMEM;
 213         }
 214
 215         dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
 216
 217         return 0;
 218 }
 219
 220 /* Map state->caam_ctx, and add it to link table */
 221 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
 222                                      struct caam_hash_state *state, int ctx_len,
 223                                      struct sec4_sg_entry *sec4_sg, u32 flag)
 224 {
 225         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
 226         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 227                 dev_err(jrdev, "unable to map ctx\n");
 228                 state->ctx_dma = 0;
 229                 return -ENOMEM;
 230         }
 231
 232         dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * For ahash update, final and finup (import_ctx = true)
 239  *     import context, read and write to seqout
 240  * For ahash firsts and digest (import_ctx = false)
 241  *     read and write to seqout
 242  */
 243 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
 244                                      struct caam_hash_ctx *ctx, bool import_ctx)
 245 {
 246         u32 op = ctx->adata.algtype;
 247         u32 *skip_key_load;
 248
 249         init_sh_desc(desc, HDR_SHARE_SERIAL);
 250
 251         /* Append key if it has been set; ahash update excluded */
 252         if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
 253                 /* Skip key loading if already shared */
 254                 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 255                                             JUMP_COND_SHRD);
 256
 257                 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
 258                                   ctx->adata.keylen, CLASS_2 |
 259                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 260
 261                 set_jump_tgt_here(desc, skip_key_load);
 262
 263                 op |= OP_ALG_AAI_HMAC_PRECOMP;
 264         }
 265
 266         /* If needed, import context from software */
 267         if (import_ctx)
 268                 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
 269                                 LDST_SRCDST_BYTE_CONTEXT);
 270
 271         /* Class 2 operation */
 272         append_operation(desc, op | state | OP_ALG_ENCRYPT);
 273
 274         /*
 275          * Load from buf and/or src and write to req->result or state->context
 276          * Calculate remaining bytes to read
 277          */
 278         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 279         /* Read remaining bytes */
 280         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
 281                              FIFOLD_TYPE_MSG | KEY_VLF);
 282         /* Store class2 context bytes */
 283         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 284                          LDST_SRCDST_BYTE_CONTEXT);
 285 }
 286
 287 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 288 {
 289         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 290         int digestsize = crypto_ahash_digestsize(ahash);
 291         struct device *jrdev = ctx->jrdev;
 292         u32 *desc;
 293
 294         /* ahash_update shared descriptor */
 295         desc = ctx->sh_desc_update;
 296         ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
 297         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
 298                                    desc_bytes(desc), DMA_TO_DEVICE);
 299 #ifdef DEBUG
 300         print_hex_dump(KERN_ERR,
 301                        "ahash update shdesc@"__stringify(__LINE__)": ",
 302                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 303 #endif
 304
 305         /* ahash_update_first shared descriptor */
 306         desc = ctx->sh_desc_update_first;
 307         ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
 308         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
 309                                    desc_bytes(desc), DMA_TO_DEVICE);
 310 #ifdef DEBUG
 311         print_hex_dump(KERN_ERR,
 312                        "ahash update first shdesc@"__stringify(__LINE__)": ",
 313                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 314 #endif
 315
 316         /* ahash_final shared descriptor */
 317         desc = ctx->sh_desc_fin;
 318         ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
 319         dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
 320                                    desc_bytes(desc), DMA_TO_DEVICE);
 321 #ifdef DEBUG
 322         print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
 323                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 324                        desc_bytes(desc), 1);
 325 #endif
 326
 327         /* ahash_digest shared descriptor */
 328         desc = ctx->sh_desc_digest;
 329         ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
 330         dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
 331                                    desc_bytes(desc), DMA_TO_DEVICE);
 332 #ifdef DEBUG
 333         print_hex_dump(KERN_ERR,
 334                        "ahash digest shdesc@"__stringify(__LINE__)": ",
 335                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 336                        desc_bytes(desc), 1);
 337 #endif
 338
 339         return 0;
 340 }
 341
 342 /* Digest hash size if it is too large */
 343 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 344                            u32 *keylen, u8 *key_out, u32 digestsize)
 345 {
 346         struct device *jrdev = ctx->jrdev;
 347         u32 *desc;
 348         struct split_key_result result;
 349         dma_addr_t src_dma, dst_dma;
 350         int ret;
 351
 352         desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 353         if (!desc) {
 354                 dev_err(jrdev, "unable to allocate key input memory\n");
 355                 return -ENOMEM;
 356         }
 357
 358         init_job_desc(desc, 0);
 359
 360         src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
 361                                  DMA_TO_DEVICE);
 362         if (dma_mapping_error(jrdev, src_dma)) {
 363                 dev_err(jrdev, "unable to map key input memory\n");
 364                 kfree(desc);
 365                 return -ENOMEM;
 366         }
 367         dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
 368                                  DMA_FROM_DEVICE);
 369         if (dma_mapping_error(jrdev, dst_dma)) {
 370                 dev_err(jrdev, "unable to map key output memory\n");
 371                 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 372                 kfree(desc);
 373                 return -ENOMEM;
 374         }
 375
 376         /* Job descriptor to perform unkeyed hash on key_in */
 377         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
 378                          OP_ALG_AS_INITFINAL);
 379         append_seq_in_ptr(desc, src_dma, *keylen, 0);
 380         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
 381                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
 382         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 383         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 384                          LDST_SRCDST_BYTE_CONTEXT);
 385
 386 #ifdef DEBUG
 387         print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
 388                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
 389         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 390                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 391 #endif
 392
 393         result.err = 0;
 394         init_completion(&result.completion);
 395
 396         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 397         if (!ret) {
 398                 /* in progress */
 399                 wait_for_completion(&result.completion);
 400                 ret = result.err;
 401 #ifdef DEBUG
 402                 print_hex_dump(KERN_ERR,
 403                                "digested key@"__stringify(__LINE__)": ",
 404                                DUMP_PREFIX_ADDRESS, 16, 4, key_in,
 405                                digestsize, 1);
 406 #endif
 407         }
 408         dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 409         dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
 410
 411         *keylen = digestsize;
 412
 413         kfree(desc);
 414
 415         return ret;
 416 }
 417
 418 static int ahash_setkey(struct crypto_ahash *ahash,
 419                         const u8 *key, unsigned int keylen)
 420 {
 421         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 422         int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 423         int digestsize = crypto_ahash_digestsize(ahash);
 424         int ret;
 425         u8 *hashed_key = NULL;
 426
 427 #ifdef DEBUG
 428         printk(KERN_ERR "keylen %d\n", keylen);
 429 #endif
 430
 431         if (keylen > blocksize) {
 432                 hashed_key = kmalloc_array(digestsize,
 433                                            sizeof(*hashed_key),
 434                                            GFP_KERNEL | GFP_DMA);
 435                 if (!hashed_key)
 436                         return -ENOMEM;
 437                 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
 438                                       digestsize);
 439                 if (ret)
 440                         goto bad_free_key;
 441                 key = hashed_key;
 442         }
 443
 444         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
 445                             CAAM_MAX_HASH_KEY_SIZE);
 446         if (ret)
 447                 goto bad_free_key;
 448
 449 #ifdef DEBUG
 450         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 451                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 452                        ctx->adata.keylen_pad, 1);
 453 #endif
 454
 455         kfree(hashed_key);
 456         return ahash_set_sh_desc(ahash);
 457  bad_free_key:
 458         kfree(hashed_key);
 459         crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 460         return -EINVAL;
 461 }
 462
 463 /*
 464  * ahash_edesc - s/w-extended ahash descriptor
 465  * @dst_dma: physical mapped address of req->result
 466  * @sec4_sg_dma: physical mapped address of h/w link table
 467  * @src_nents: number of segments in input scatterlist
 468  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 469  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 470  * @sec4_sg: h/w link table
 471  */
 472 struct ahash_edesc {
 473         dma_addr_t dst_dma;
 474         dma_addr_t sec4_sg_dma;
 475         int src_nents;
 476         int sec4_sg_bytes;
 477         u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
 478         struct sec4_sg_entry sec4_sg[0];
 479 };
 480
 481 static inline void ahash_unmap(struct device *dev,
 482                         struct ahash_edesc *edesc,
 483                         struct ahash_request *req, int dst_len)
 484 {
 485         struct caam_hash_state *state = ahash_request_ctx(req);
 486
 487         if (edesc->src_nents)
 488                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
 489         if (edesc->dst_dma)
 490                 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
 491
 492         if (edesc->sec4_sg_bytes)
 493                 dma_unmap_single(dev, edesc->sec4_sg_dma,
 494                                  edesc->sec4_sg_bytes, DMA_TO_DEVICE);
 495
 496         if (state->buf_dma) {
 497                 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
 498                                  DMA_TO_DEVICE);
 499                 state->buf_dma = 0;
 500         }
 501 }
 502
 503 static inline void ahash_unmap_ctx(struct device *dev,
 504                         struct ahash_edesc *edesc,
 505                         struct ahash_request *req, int dst_len, u32 flag)
 506 {
 507         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 508         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 509         struct caam_hash_state *state = ahash_request_ctx(req);
 510
 511         if (state->ctx_dma) {
 512                 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
 513                 state->ctx_dma = 0;
 514         }
 515         ahash_unmap(dev, edesc, req, dst_len);
 516 }
 517
 518 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
 519                        void *context)
 520 {
 521         struct ahash_request *req = context;
 522         struct ahash_edesc *edesc;
 523         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 524         int digestsize = crypto_ahash_digestsize(ahash);
 525 #ifdef DEBUG
 526         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 527         struct caam_hash_state *state = ahash_request_ctx(req);
 528
 529         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 530 #endif
 531
 532         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 533         if (err)
 534                 caam_jr_strstatus(jrdev, err);
 535
 536         ahash_unmap(jrdev, edesc, req, digestsize);
 537         kfree(edesc);
 538
 539 #ifdef DEBUG
 540         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 541                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 542                        ctx->ctx_len, 1);
 543         if (req->result)
 544                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 545                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 546                                digestsize, 1);
 547 #endif
 548
 549         req->base.complete(&req->base, err);
 550 }
 551
 552 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
 553                             void *context)
 554 {
 555         struct ahash_request *req = context;
 556         struct ahash_edesc *edesc;
 557         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 558         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 559         struct caam_hash_state *state = ahash_request_ctx(req);
 560 #ifdef DEBUG
 561         int digestsize = crypto_ahash_digestsize(ahash);
 562
 563         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 564 #endif
 565
 566         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 567         if (err)
 568                 caam_jr_strstatus(jrdev, err);
 569
 570         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 571         switch_buf(state);
 572         kfree(edesc);
 573
 574 #ifdef DEBUG
 575         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 576                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 577                        ctx->ctx_len, 1);
 578         if (req->result)
 579                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 580                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 581                                digestsize, 1);
 582 #endif
 583
 584         req->base.complete(&req->base, err);
 585 }
 586
 587 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
 588                                void *context)
 589 {
 590         struct ahash_request *req = context;
 591         struct ahash_edesc *edesc;
 592         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 593         int digestsize = crypto_ahash_digestsize(ahash);
 594 #ifdef DEBUG
 595         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 596         struct caam_hash_state *state = ahash_request_ctx(req);
 597
 598         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 599 #endif
 600
 601         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 602         if (err)
 603                 caam_jr_strstatus(jrdev, err);
 604
 605         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
 606         kfree(edesc);
 607
 608 #ifdef DEBUG
 609         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 610                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 611                        ctx->ctx_len, 1);
 612         if (req->result)
 613                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 614                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 615                                digestsize, 1);
 616 #endif
 617
 618         req->base.complete(&req->base, err);
 619 }
 620
 621 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 622                                void *context)
 623 {
 624         struct ahash_request *req = context;
 625         struct ahash_edesc *edesc;
 626         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 627         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 628         struct caam_hash_state *state = ahash_request_ctx(req);
 629 #ifdef DEBUG
 630         int digestsize = crypto_ahash_digestsize(ahash);
 631
 632         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 633 #endif
 634
 635         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 636         if (err)
 637                 caam_jr_strstatus(jrdev, err);
 638
 639         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
 640         switch_buf(state);
 641         kfree(edesc);
 642
 643 #ifdef DEBUG
 644         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 645                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 646                        ctx->ctx_len, 1);
 647         if (req->result)
 648                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 649                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 650                                digestsize, 1);
 651 #endif
 652
 653         req->base.complete(&req->base, err);
 654 }
 655
 656 /*
 657  * Allocate an enhanced descriptor, which contains the hardware descriptor
 658  * and space for hardware scatter table containing sg_num entries.
 659  */
 660 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
 661                                              int sg_num, u32 *sh_desc,
 662                                              dma_addr_t sh_desc_dma,
 663                                              gfp_t flags)
 664 {
 665         struct ahash_edesc *edesc;
 666         unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
 667
 668         edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
 669         if (!edesc) {
 670                 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
 671                 return NULL;
 672         }
 673
 674         init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
 675                              HDR_SHARE_DEFER | HDR_REVERSE);
 676
 677         return edesc;
 678 }
 679
 680 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
 681                                struct ahash_edesc *edesc,
 682                                struct ahash_request *req, int nents,
 683                                unsigned int first_sg,
 684                                unsigned int first_bytes, size_t to_hash)
 685 {
 686         dma_addr_t src_dma;
 687         u32 options;
 688
 689         if (nents > 1 || first_sg) {
 690                 struct sec4_sg_entry *sg = edesc->sec4_sg;
 691                 unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
 692
 693                 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
 694
 695                 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
 696                 if (dma_mapping_error(ctx->jrdev, src_dma)) {
 697                         dev_err(ctx->jrdev, "unable to map S/G table\n");
 698                         return -ENOMEM;
 699                 }
 700
 701                 edesc->sec4_sg_bytes = sgsize;
 702                 edesc->sec4_sg_dma = src_dma;
 703                 options = LDST_SGF;
 704         } else {
 705                 src_dma = sg_dma_address(req->src);
 706                 options = 0;
 707         }
 708
 709         append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
 710                           options);
 711
 712         return 0;
 713 }
 714
 715 /* submit update job descriptor */
 716 static int ahash_update_ctx(struct ahash_request *req)
 717 {
 718         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 719         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 720         struct caam_hash_state *state = ahash_request_ctx(req);
 721         struct device *jrdev = ctx->jrdev;
 722         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 723                        GFP_KERNEL : GFP_ATOMIC;
 724         u8 *buf = current_buf(state);
 725         int *buflen = current_buflen(state);
 726         u8 *next_buf = alt_buf(state);
 727         int *next_buflen = alt_buflen(state), last_buflen;
 728         int in_len = *buflen + req->nbytes, to_hash;
 729         u32 *desc;
 730         int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
 731         struct ahash_edesc *edesc;
 732         int ret = 0;
 733
 734         last_buflen = *next_buflen;
 735         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
 736         to_hash = in_len - *next_buflen;
 737
 738         if (to_hash) {
 739                 src_nents = sg_nents_for_len(req->src,
 740                                              req->nbytes - (*next_buflen));
 741                 if (src_nents < 0) {
 742                         dev_err(jrdev, "Invalid number of src SG.\n");
 743                         return src_nents;
 744                 }
 745
 746                 if (src_nents) {
 747                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 748                                                   DMA_TO_DEVICE);
 749                         if (!mapped_nents) {
 750                                 dev_err(jrdev, "unable to DMA map source\n");
 751                                 return -ENOMEM;
 752                         }
 753                 } else {
 754                         mapped_nents = 0;
 755                 }
 756
 757                 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
 758                 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 759                                  sizeof(struct sec4_sg_entry);
 760
 761                 /*
 762                  * allocate space for base edesc and hw desc commands,
 763                  * link tables
 764                  */
 765                 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 766                                           ctx->sh_desc_update,
 767                                           ctx->sh_desc_update_dma, flags);
 768                 if (!edesc) {
 769                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 770                         return -ENOMEM;
 771                 }
 772
 773                 edesc->src_nents = src_nents;
 774                 edesc->sec4_sg_bytes = sec4_sg_bytes;
 775
 776                 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 777                                          edesc->sec4_sg, DMA_BIDIRECTIONAL);
 778                 if (ret)
 779                         goto unmap_ctx;
 780
 781                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 782                 if (ret)
 783                         goto unmap_ctx;
 784
 785                 if (mapped_nents) {
 786                         sg_to_sec4_sg_last(req->src, mapped_nents,
 787                                            edesc->sec4_sg + sec4_sg_src_index,
 788                                            0);
 789                         if (*next_buflen)
 790                                 scatterwalk_map_and_copy(next_buf, req->src,
 791                                                          to_hash - *buflen,
 792                                                          *next_buflen, 0);
 793                 } else {
 794                         sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
 795                                             1);
 796                 }
 797
 798                 desc = edesc->hw_desc;
 799
 800                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 801                                                      sec4_sg_bytes,
 802                                                      DMA_TO_DEVICE);
 803                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 804                         dev_err(jrdev, "unable to map S/G table\n");
 805                         ret = -ENOMEM;
 806                         goto unmap_ctx;
 807                 }
 808
 809                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
 810                                        to_hash, LDST_SGF);
 811
 812                 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
 813
 814 #ifdef DEBUG
 815                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 816                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
 817                                desc_bytes(desc), 1);
 818 #endif
 819
 820                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
 821                 if (ret)
 822                         goto unmap_ctx;
 823
 824                 ret = -EINPROGRESS;
 825         } else if (*next_buflen) {
 826                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 827                                          req->nbytes, 0);
 828                 *buflen = *next_buflen;
 829                 *next_buflen = last_buflen;
 830         }
 831 #ifdef DEBUG
 832         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
 833                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
 834         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
 835                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
 836                        *next_buflen, 1);
 837 #endif
 838
 839         return ret;
 840  unmap_ctx:
 841         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 842         kfree(edesc);
 843         return ret;
 844 }
 845
 846 static int ahash_final_ctx(struct ahash_request *req)
 847 {
 848         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 849         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 850         struct caam_hash_state *state = ahash_request_ctx(req);
 851         struct device *jrdev = ctx->jrdev;
 852         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 853                        GFP_KERNEL : GFP_ATOMIC;
 854         int buflen = *current_buflen(state);
 855         u32 *desc;
 856         int sec4_sg_bytes, sec4_sg_src_index;
 857         int digestsize = crypto_ahash_digestsize(ahash);
 858         struct ahash_edesc *edesc;
 859         int ret;
 860
 861         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 862         sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
 863
 864         /* allocate space for base edesc and hw desc commands, link tables */
 865         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
 866                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 867                                   flags);
 868         if (!edesc)
 869                 return -ENOMEM;
 870
 871         desc = edesc->hw_desc;
 872
 873         edesc->sec4_sg_bytes = sec4_sg_bytes;
 874         edesc->src_nents = 0;
 875
 876         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 877                                  edesc->sec4_sg, DMA_TO_DEVICE);
 878         if (ret)
 879                 goto unmap_ctx;
 880
 881         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 882         if (ret)
 883                 goto unmap_ctx;
 884
 885         sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
 886
 887         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 888                                             sec4_sg_bytes, DMA_TO_DEVICE);
 889         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 890                 dev_err(jrdev, "unable to map S/G table\n");
 891                 ret = -ENOMEM;
 892                 goto unmap_ctx;
 893         }
 894
 895         append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
 896                           LDST_SGF);
 897
 898         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 899                                                 digestsize);
 900         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 901                 dev_err(jrdev, "unable to map dst\n");
 902                 ret = -ENOMEM;
 903                 goto unmap_ctx;
 904         }
 905
 906 #ifdef DEBUG
 907         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 908                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 909 #endif
 910
 911         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
 912         if (ret)
 913                 goto unmap_ctx;
 914
 915         return -EINPROGRESS;
 916  unmap_ctx:
 917         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
 918         kfree(edesc);
 919         return ret;
 920 }
 921
 922 static int ahash_finup_ctx(struct ahash_request *req)
 923 {
 924         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 925         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 926         struct caam_hash_state *state = ahash_request_ctx(req);
 927         struct device *jrdev = ctx->jrdev;
 928         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 929                        GFP_KERNEL : GFP_ATOMIC;
 930         int buflen = *current_buflen(state);
 931         u32 *desc;
 932         int sec4_sg_src_index;
 933         int src_nents, mapped_nents;
 934         int digestsize = crypto_ahash_digestsize(ahash);
 935         struct ahash_edesc *edesc;
 936         int ret;
 937
 938         src_nents = sg_nents_for_len(req->src, req->nbytes);
 939         if (src_nents < 0) {
 940                 dev_err(jrdev, "Invalid number of src SG.\n");
 941                 return src_nents;
 942         }
 943
 944         if (src_nents) {
 945                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 946                                           DMA_TO_DEVICE);
 947                 if (!mapped_nents) {
 948                         dev_err(jrdev, "unable to DMA map source\n");
 949                         return -ENOMEM;
 950                 }
 951         } else {
 952                 mapped_nents = 0;
 953         }
 954
 955         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 956
 957         /* allocate space for base edesc and hw desc commands, link tables */
 958         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 959                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 960                                   flags);
 961         if (!edesc) {
 962                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 963                 return -ENOMEM;
 964         }
 965
 966         desc = edesc->hw_desc;
 967
 968         edesc->src_nents = src_nents;
 969
 970         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 971                                  edesc->sec4_sg, DMA_TO_DEVICE);
 972         if (ret)
 973                 goto unmap_ctx;
 974
 975         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 976         if (ret)
 977                 goto unmap_ctx;
 978
 979         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
 980                                   sec4_sg_src_index, ctx->ctx_len + buflen,
 981                                   req->nbytes);
 982         if (ret)
 983                 goto unmap_ctx;
 984
 985         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 986                                                 digestsize);
 987         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 988                 dev_err(jrdev, "unable to map dst\n");
 989                 ret = -ENOMEM;
 990                 goto unmap_ctx;
 991         }
 992
 993 #ifdef DEBUG
 994         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 995                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 996 #endif
 997
 998         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
 999         if (ret)
1000                 goto unmap_ctx;
1001
1002         return -EINPROGRESS;
1003  unmap_ctx:
1004         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1005         kfree(edesc);
1006         return ret;
1007 }
1008
1009 static int ahash_digest(struct ahash_request *req)
1010 {
1011         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1012         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1013         struct caam_hash_state *state = ahash_request_ctx(req);
1014         struct device *jrdev = ctx->jrdev;
1015         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1016                        GFP_KERNEL : GFP_ATOMIC;
1017         u32 *desc;
1018         int digestsize = crypto_ahash_digestsize(ahash);
1019         int src_nents, mapped_nents;
1020         struct ahash_edesc *edesc;
1021         int ret;
1022
1023         state->buf_dma = 0;
1024
1025         src_nents = sg_nents_for_len(req->src, req->nbytes);
1026         if (src_nents < 0) {
1027                 dev_err(jrdev, "Invalid number of src SG.\n");
1028                 return src_nents;
1029         }
1030
1031         if (src_nents) {
1032                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1033                                           DMA_TO_DEVICE);
1034                 if (!mapped_nents) {
1035                         dev_err(jrdev, "unable to map source for DMA\n");
1036                         return -ENOMEM;
1037                 }
1038         } else {
1039                 mapped_nents = 0;
1040         }
1041
1042         /* allocate space for base edesc and hw desc commands, link tables */
1043         edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1044                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1045                                   flags);
1046         if (!edesc) {
1047                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1048                 return -ENOMEM;
1049         }
1050
1051         edesc->src_nents = src_nents;
1052
1053         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1054                                   req->nbytes);
1055         if (ret) {
1056                 ahash_unmap(jrdev, edesc, req, digestsize);
1057                 kfree(edesc);
1058                 return ret;
1059         }
1060
1061         desc = edesc->hw_desc;
1062
1063         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1064                                                 digestsize);
1065         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1066                 dev_err(jrdev, "unable to map dst\n");
1067                 ahash_unmap(jrdev, edesc, req, digestsize);
1068                 kfree(edesc);
1069                 return -ENOMEM;
1070         }
1071
1072 #ifdef DEBUG
1073         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1074                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1075 #endif
1076
1077         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1078         if (!ret) {
1079                 ret = -EINPROGRESS;
1080         } else {
1081                 ahash_unmap(jrdev, edesc, req, digestsize);
1082                 kfree(edesc);
1083         }
1084
1085         return ret;
1086 }
1087
1088 /* submit ahash final if it the first job descriptor */
1089 static int ahash_final_no_ctx(struct ahash_request *req)
1090 {
1091         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1092         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1093         struct caam_hash_state *state = ahash_request_ctx(req);
1094         struct device *jrdev = ctx->jrdev;
1095         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1096                        GFP_KERNEL : GFP_ATOMIC;
1097         u8 *buf = current_buf(state);
1098         int buflen = *current_buflen(state);
1099         u32 *desc;
1100         int digestsize = crypto_ahash_digestsize(ahash);
1101         struct ahash_edesc *edesc;
1102         int ret;
1103
1104         /* allocate space for base edesc and hw desc commands, link tables */
1105         edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1106                                   ctx->sh_desc_digest_dma, flags);
1107         if (!edesc)
1108                 return -ENOMEM;
1109
1110         desc = edesc->hw_desc;
1111
1112         if (buflen) {
1113                 state->buf_dma = dma_map_single(jrdev, buf, buflen,
1114                                                 DMA_TO_DEVICE);
1115                 if (dma_mapping_error(jrdev, state->buf_dma)) {
1116                         dev_err(jrdev, "unable to map src\n");
1117                         goto unmap;
1118                 }
1119
1120                 append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1121         }
1122
1123         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1124                                                 digestsize);
1125         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1126                 dev_err(jrdev, "unable to map dst\n");
1127                 goto unmap;
1128         }
1129         edesc->src_nents = 0;
1130
1131 #ifdef DEBUG
1132         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1133                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1134 #endif
1135
1136         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1137         if (!ret) {
1138                 ret = -EINPROGRESS;
1139         } else {
1140                 ahash_unmap(jrdev, edesc, req, digestsize);
1141                 kfree(edesc);
1142         }
1143
1144         return ret;
1145  unmap:
1146         ahash_unmap(jrdev, edesc, req, digestsize);
1147         kfree(edesc);
1148         return -ENOMEM;
1149
1150 }
1151
1152 /* submit ahash update if it the first job descriptor after update */
1153 static int ahash_update_no_ctx(struct ahash_request *req)
1154 {
1155         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1156         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1157         struct caam_hash_state *state = ahash_request_ctx(req);
1158         struct device *jrdev = ctx->jrdev;
1159         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1160                        GFP_KERNEL : GFP_ATOMIC;
1161         u8 *buf = current_buf(state);
1162         int *buflen = current_buflen(state);
1163         u8 *next_buf = alt_buf(state);
1164         int *next_buflen = alt_buflen(state);
1165         int in_len = *buflen + req->nbytes, to_hash;
1166         int sec4_sg_bytes, src_nents, mapped_nents;
1167         struct ahash_edesc *edesc;
1168         u32 *desc;
1169         int ret = 0;
1170
1171         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1172         to_hash = in_len - *next_buflen;
1173
1174         if (to_hash) {
1175                 src_nents = sg_nents_for_len(req->src,
1176                                              req->nbytes - *next_buflen);
1177                 if (src_nents < 0) {
1178                         dev_err(jrdev, "Invalid number of src SG.\n");
1179                         return src_nents;
1180                 }
1181
1182                 if (src_nents) {
1183                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1184                                                   DMA_TO_DEVICE);
1185                         if (!mapped_nents) {
1186                                 dev_err(jrdev, "unable to DMA map source\n");
1187                                 return -ENOMEM;
1188                         }
1189                 } else {
1190                         mapped_nents = 0;
1191                 }
1192
1193                 sec4_sg_bytes = (1 + mapped_nents) *
1194                                 sizeof(struct sec4_sg_entry);
1195
1196                 /*
1197                  * allocate space for base edesc and hw desc commands,
1198                  * link tables
1199                  */
1200                 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
1201                                           ctx->sh_desc_update_first,
1202                                           ctx->sh_desc_update_first_dma,
1203                                           flags);
1204                 if (!edesc) {
1205                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1206                         return -ENOMEM;
1207                 }
1208
1209                 edesc->src_nents = src_nents;
1210                 edesc->sec4_sg_bytes = sec4_sg_bytes;
1211                 edesc->dst_dma = 0;
1212
1213                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1214                 if (ret)
1215                         goto unmap_ctx;
1216
1217                 sg_to_sec4_sg_last(req->src, mapped_nents,
1218                                    edesc->sec4_sg + 1, 0);
1219
1220                 if (*next_buflen) {
1221                         scatterwalk_map_and_copy(next_buf, req->src,
1222                                                  to_hash - *buflen,
1223                                                  *next_buflen, 0);
1224                 }
1225
1226                 desc = edesc->hw_desc;
1227
1228                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1229                                                     sec4_sg_bytes,
1230                                                     DMA_TO_DEVICE);
1231                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1232                         dev_err(jrdev, "unable to map S/G table\n");
1233                         ret = -ENOMEM;
1234                         goto unmap_ctx;
1235                 }
1236
1237                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1238
1239                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1240                 if (ret)
1241                         goto unmap_ctx;
1242
1243 #ifdef DEBUG
1244                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1245                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1246                                desc_bytes(desc), 1);
1247 #endif
1248
1249                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1250                 if (ret)
1251                         goto unmap_ctx;
1252
1253                 ret = -EINPROGRESS;
1254                 state->update = ahash_update_ctx;
1255                 state->finup = ahash_finup_ctx;
1256                 state->final = ahash_final_ctx;
1257         } else if (*next_buflen) {
1258                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1259                                          req->nbytes, 0);
1260                 *buflen = *next_buflen;
1261                 *next_buflen = 0;
1262         }
1263 #ifdef DEBUG
1264         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1265                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1266         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1267                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1268                        *next_buflen, 1);
1269 #endif
1270
1271         return ret;
1272  unmap_ctx:
1273         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1274         kfree(edesc);
1275         return ret;
1276 }
1277
1278 /* submit ahash finup if it the first job descriptor after update */
1279 static int ahash_finup_no_ctx(struct ahash_request *req)
1280 {
1281         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1282         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1283         struct caam_hash_state *state = ahash_request_ctx(req);
1284         struct device *jrdev = ctx->jrdev;
1285         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1286                        GFP_KERNEL : GFP_ATOMIC;
1287         int buflen = *current_buflen(state);
1288         u32 *desc;
1289         int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1290         int digestsize = crypto_ahash_digestsize(ahash);
1291         struct ahash_edesc *edesc;
1292         int ret;
1293
1294         src_nents = sg_nents_for_len(req->src, req->nbytes);
1295         if (src_nents < 0) {
1296                 dev_err(jrdev, "Invalid number of src SG.\n");
1297                 return src_nents;
1298         }
1299
1300         if (src_nents) {
1301                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1302                                           DMA_TO_DEVICE);
1303                 if (!mapped_nents) {
1304                         dev_err(jrdev, "unable to DMA map source\n");
1305                         return -ENOMEM;
1306                 }
1307         } else {
1308                 mapped_nents = 0;
1309         }
1310
1311         sec4_sg_src_index = 2;
1312         sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1313                          sizeof(struct sec4_sg_entry);
1314
1315         /* allocate space for base edesc and hw desc commands, link tables */
1316         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1317                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1318                                   flags);
1319         if (!edesc) {
1320                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1321                 return -ENOMEM;
1322         }
1323
1324         desc = edesc->hw_desc;
1325
1326         edesc->src_nents = src_nents;
1327         edesc->sec4_sg_bytes = sec4_sg_bytes;
1328
1329         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1330         if (ret)
1331                 goto unmap;
1332
1333         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1334                                   req->nbytes);
1335         if (ret) {
1336                 dev_err(jrdev, "unable to map S/G table\n");
1337                 goto unmap;
1338         }
1339
1340         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1341                                                 digestsize);
1342         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1343                 dev_err(jrdev, "unable to map dst\n");
1344                 goto unmap;
1345         }
1346
1347 #ifdef DEBUG
1348         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1349                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1350 #endif
1351
1352         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1353         if (!ret) {
1354                 ret = -EINPROGRESS;
1355         } else {
1356                 ahash_unmap(jrdev, edesc, req, digestsize);
1357                 kfree(edesc);
1358         }
1359
1360         return ret;
1361  unmap:
1362         ahash_unmap(jrdev, edesc, req, digestsize);
1363         kfree(edesc);
1364         return -ENOMEM;
1365
1366 }
1367
1368 /* submit first update job descriptor after init */
1369 static int ahash_update_first(struct ahash_request *req)
1370 {
1371         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1372         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1373         struct caam_hash_state *state = ahash_request_ctx(req);
1374         struct device *jrdev = ctx->jrdev;
1375         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1376                        GFP_KERNEL : GFP_ATOMIC;
1377         u8 *next_buf = alt_buf(state);
1378         int *next_buflen = alt_buflen(state);
1379         int to_hash;
1380         u32 *desc;
1381         int src_nents, mapped_nents;
1382         struct ahash_edesc *edesc;
1383         int ret = 0;
1384
1385         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1386                                       1);
1387         to_hash = req->nbytes - *next_buflen;
1388
1389         if (to_hash) {
1390                 src_nents = sg_nents_for_len(req->src,
1391                                              req->nbytes - *next_buflen);
1392                 if (src_nents < 0) {
1393                         dev_err(jrdev, "Invalid number of src SG.\n");
1394                         return src_nents;
1395                 }
1396
1397                 if (src_nents) {
1398                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1399                                                   DMA_TO_DEVICE);
1400                         if (!mapped_nents) {
1401                                 dev_err(jrdev, "unable to map source for DMA\n");
1402                                 return -ENOMEM;
1403                         }
1404                 } else {
1405                         mapped_nents = 0;
1406                 }
1407
1408                 /*
1409                  * allocate space for base edesc and hw desc commands,
1410                  * link tables
1411                  */
1412                 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1413                                           mapped_nents : 0,
1414                                           ctx->sh_desc_update_first,
1415                                           ctx->sh_desc_update_first_dma,
1416                                           flags);
1417                 if (!edesc) {
1418                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1419                         return -ENOMEM;
1420                 }
1421
1422                 edesc->src_nents = src_nents;
1423                 edesc->dst_dma = 0;
1424
1425                 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1426                                           to_hash);
1427                 if (ret)
1428                         goto unmap_ctx;
1429
1430                 if (*next_buflen)
1431                         scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1432                                                  *next_buflen, 0);
1433
1434                 desc = edesc->hw_desc;
1435
1436                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1437                 if (ret)
1438                         goto unmap_ctx;
1439
1440 #ifdef DEBUG
1441                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1442                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1443                                desc_bytes(desc), 1);
1444 #endif
1445
1446                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1447                 if (ret)
1448                         goto unmap_ctx;
1449
1450                 ret = -EINPROGRESS;
1451                 state->update = ahash_update_ctx;
1452                 state->finup = ahash_finup_ctx;
1453                 state->final = ahash_final_ctx;
1454         } else if (*next_buflen) {
1455                 state->update = ahash_update_no_ctx;
1456                 state->finup = ahash_finup_no_ctx;
1457                 state->final = ahash_final_no_ctx;
1458                 scatterwalk_map_and_copy(next_buf, req->src, 0,
1459                                          req->nbytes, 0);
1460                 switch_buf(state);
1461         }
1462 #ifdef DEBUG
1463         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1464                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1465                        *next_buflen, 1);
1466 #endif
1467
1468         return ret;
1469  unmap_ctx:
1470         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1471         kfree(edesc);
1472         return ret;
1473 }
1474
1475 static int ahash_finup_first(struct ahash_request *req)
1476 {
1477         return ahash_digest(req);
1478 }
1479
1480 static int ahash_init(struct ahash_request *req)
1481 {
1482         struct caam_hash_state *state = ahash_request_ctx(req);
1483
1484         state->update = ahash_update_first;
1485         state->finup = ahash_finup_first;
1486         state->final = ahash_final_no_ctx;
1487
1488         state->ctx_dma = 0;
1489         state->current_buf = 0;
1490         state->buf_dma = 0;
1491         state->buflen_0 = 0;
1492         state->buflen_1 = 0;
1493
1494         return 0;
1495 }
1496
1497 static int ahash_update(struct ahash_request *req)
1498 {
1499         struct caam_hash_state *state = ahash_request_ctx(req);
1500
1501         return state->update(req);
1502 }
1503
1504 static int ahash_finup(struct ahash_request *req)
1505 {
1506         struct caam_hash_state *state = ahash_request_ctx(req);
1507
1508         return state->finup(req);
1509 }
1510
1511 static int ahash_final(struct ahash_request *req)
1512 {
1513         struct caam_hash_state *state = ahash_request_ctx(req);
1514
1515         return state->final(req);
1516 }
1517
1518 static int ahash_export(struct ahash_request *req, void *out)
1519 {
1520         struct caam_hash_state *state = ahash_request_ctx(req);
1521         struct caam_export_state *export = out;
1522         int len;
1523         u8 *buf;
1524
1525         if (state->current_buf) {
1526                 buf = state->buf_1;
1527                 len = state->buflen_1;
1528         } else {
1529                 buf = state->buf_0;
1530                 len = state->buflen_0;
1531         }
1532
1533         memcpy(export->buf, buf, len);
1534         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1535         export->buflen = len;
1536         export->update = state->update;
1537         export->final = state->final;
1538         export->finup = state->finup;
1539
1540         return 0;
1541 }
1542
1543 static int ahash_import(struct ahash_request *req, const void *in)
1544 {
1545         struct caam_hash_state *state = ahash_request_ctx(req);
1546         const struct caam_export_state *export = in;
1547
1548         memset(state, 0, sizeof(*state));
1549         memcpy(state->buf_0, export->buf, export->buflen);
1550         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1551         state->buflen_0 = export->buflen;
1552         state->update = export->update;
1553         state->final = export->final;
1554         state->finup = export->finup;
1555
1556         return 0;
1557 }
1558
1559 struct caam_hash_template {
1560         char name[CRYPTO_MAX_ALG_NAME];
1561         char driver_name[CRYPTO_MAX_ALG_NAME];
1562         char hmac_name[CRYPTO_MAX_ALG_NAME];
1563         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1564         unsigned int blocksize;
1565         struct ahash_alg template_ahash;
1566         u32 alg_type;
1567 };
1568
1569 /* ahash descriptors */
1570 static struct caam_hash_template driver_hash[] = {
1571         {
1572                 .name = "sha1",
1573                 .driver_name = "sha1-caam",
1574                 .hmac_name = "hmac(sha1)",
1575                 .hmac_driver_name = "hmac-sha1-caam",
1576                 .blocksize = SHA1_BLOCK_SIZE,
1577                 .template_ahash = {
1578                         .init = ahash_init,
1579                         .update = ahash_update,
1580                         .final = ahash_final,
1581                         .finup = ahash_finup,
1582                         .digest = ahash_digest,
1583                         .export = ahash_export,
1584                         .import = ahash_import,
1585                         .setkey = ahash_setkey,
1586                         .halg = {
1587                                 .digestsize = SHA1_DIGEST_SIZE,
1588                                 .statesize = sizeof(struct caam_export_state),
1589                         },
1590                 },
1591                 .alg_type = OP_ALG_ALGSEL_SHA1,
1592         }, {
1593                 .name = "sha224",
1594                 .driver_name = "sha224-caam",
1595                 .hmac_name = "hmac(sha224)",
1596                 .hmac_driver_name = "hmac-sha224-caam",
1597                 .blocksize = SHA224_BLOCK_SIZE,
1598                 .template_ahash = {
1599                         .init = ahash_init,
1600                         .update = ahash_update,
1601                         .final = ahash_final,
1602                         .finup = ahash_finup,
1603                         .digest = ahash_digest,
1604                         .export = ahash_export,
1605                         .import = ahash_import,
1606                         .setkey = ahash_setkey,
1607                         .halg = {
1608                                 .digestsize = SHA224_DIGEST_SIZE,
1609                                 .statesize = sizeof(struct caam_export_state),
1610                         },
1611                 },
1612                 .alg_type = OP_ALG_ALGSEL_SHA224,
1613         }, {
1614                 .name = "sha256",
1615                 .driver_name = "sha256-caam",
1616                 .hmac_name = "hmac(sha256)",
1617                 .hmac_driver_name = "hmac-sha256-caam",
1618                 .blocksize = SHA256_BLOCK_SIZE,
1619                 .template_ahash = {
1620                         .init = ahash_init,
1621                         .update = ahash_update,
1622                         .final = ahash_final,
1623                         .finup = ahash_finup,
1624                         .digest = ahash_digest,
1625                         .export = ahash_export,
1626                         .import = ahash_import,
1627                         .setkey = ahash_setkey,
1628                         .halg = {
1629                                 .digestsize = SHA256_DIGEST_SIZE,
1630                                 .statesize = sizeof(struct caam_export_state),
1631                         },
1632                 },
1633                 .alg_type = OP_ALG_ALGSEL_SHA256,
1634         }, {
1635                 .name = "sha384",
1636                 .driver_name = "sha384-caam",
1637                 .hmac_name = "hmac(sha384)",
1638                 .hmac_driver_name = "hmac-sha384-caam",
1639                 .blocksize = SHA384_BLOCK_SIZE,
1640                 .template_ahash = {
1641                         .init = ahash_init,
1642                         .update = ahash_update,
1643                         .final = ahash_final,
1644                         .finup = ahash_finup,
1645                         .digest = ahash_digest,
1646                         .export = ahash_export,
1647                         .import = ahash_import,
1648                         .setkey = ahash_setkey,
1649                         .halg = {
1650                                 .digestsize = SHA384_DIGEST_SIZE,
1651                                 .statesize = sizeof(struct caam_export_state),
1652                         },
1653                 },
1654                 .alg_type = OP_ALG_ALGSEL_SHA384,
1655         }, {
1656                 .name = "sha512",
1657                 .driver_name = "sha512-caam",
1658                 .hmac_name = "hmac(sha512)",
1659                 .hmac_driver_name = "hmac-sha512-caam",
1660                 .blocksize = SHA512_BLOCK_SIZE,
1661                 .template_ahash = {
1662                         .init = ahash_init,
1663                         .update = ahash_update,
1664                         .final = ahash_final,
1665                         .finup = ahash_finup,
1666                         .digest = ahash_digest,
1667                         .export = ahash_export,
1668                         .import = ahash_import,
1669                         .setkey = ahash_setkey,
1670                         .halg = {
1671                                 .digestsize = SHA512_DIGEST_SIZE,
1672                                 .statesize = sizeof(struct caam_export_state),
1673                         },
1674                 },
1675                 .alg_type = OP_ALG_ALGSEL_SHA512,
1676         }, {
1677                 .name = "md5",
1678                 .driver_name = "md5-caam",
1679                 .hmac_name = "hmac(md5)",
1680                 .hmac_driver_name = "hmac-md5-caam",
1681                 .blocksize = MD5_BLOCK_WORDS * 4,
1682                 .template_ahash = {
1683                         .init = ahash_init,
1684                         .update = ahash_update,
1685                         .final = ahash_final,
1686                         .finup = ahash_finup,
1687                         .digest = ahash_digest,
1688                         .export = ahash_export,
1689                         .import = ahash_import,
1690                         .setkey = ahash_setkey,
1691                         .halg = {
1692                                 .digestsize = MD5_DIGEST_SIZE,
1693                                 .statesize = sizeof(struct caam_export_state),
1694                         },
1695                 },
1696                 .alg_type = OP_ALG_ALGSEL_MD5,
1697         },
1698 };
1699
1700 struct caam_hash_alg {
1701         struct list_head entry;
1702         int alg_type;
1703         struct ahash_alg ahash_alg;
1704 };
1705
1706 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1707 {
1708         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1709         struct crypto_alg *base = tfm->__crt_alg;
1710         struct hash_alg_common *halg =
1711                  container_of(base, struct hash_alg_common, base);
1712         struct ahash_alg *alg =
1713                  container_of(halg, struct ahash_alg, halg);
1714         struct caam_hash_alg *caam_hash =
1715                  container_of(alg, struct caam_hash_alg, ahash_alg);
1716         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1717         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1718         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1719                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1720                                          HASH_MSG_LEN + 32,
1721                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1722                                          HASH_MSG_LEN + 64,
1723                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1724         dma_addr_t dma_addr;
1725
1726         /*
1727          * Get a Job ring from Job Ring driver to ensure in-order
1728          * crypto request processing per tfm
1729          */
1730         ctx->jrdev = caam_jr_alloc();
1731         if (IS_ERR(ctx->jrdev)) {
1732                 pr_err("Job Ring Device allocation for transform failed\n");
1733                 return PTR_ERR(ctx->jrdev);
1734         }
1735
1736         dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1737                                         offsetof(struct caam_hash_ctx,
1738                                                  sh_desc_update_dma),
1739                                         DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1740         if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1741                 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1742                 caam_jr_free(ctx->jrdev);
1743                 return -ENOMEM;
1744         }
1745
1746         ctx->sh_desc_update_dma = dma_addr;
1747         ctx->sh_desc_update_first_dma = dma_addr +
1748                                         offsetof(struct caam_hash_ctx,
1749                                                  sh_desc_update_first);
1750         ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1751                                                    sh_desc_fin);
1752         ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1753                                                       sh_desc_digest);
1754
1755         /* copy descriptor header template value */
1756         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1757
1758         ctx->ctx_len = runninglen[(ctx->adata.algtype &
1759                                    OP_ALG_ALGSEL_SUBMASK) >>
1760                                   OP_ALG_ALGSEL_SHIFT];
1761
1762         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1763                                  sizeof(struct caam_hash_state));
1764         return ahash_set_sh_desc(ahash);
1765 }
1766
1767 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1768 {
1769         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1770
1771         dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1772                                offsetof(struct caam_hash_ctx,
1773                                         sh_desc_update_dma),
1774                                DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1775         caam_jr_free(ctx->jrdev);
1776 }
1777
1778 static void __exit caam_algapi_hash_exit(void)
1779 {
1780         struct caam_hash_alg *t_alg, *n;
1781
1782         if (!hash_list.next)
1783                 return;
1784
1785         list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1786                 crypto_unregister_ahash(&t_alg->ahash_alg);
1787                 list_del(&t_alg->entry);
1788                 kfree(t_alg);
1789         }
1790 }
1791
1792 static struct caam_hash_alg *
1793 caam_hash_alloc(struct caam_hash_template *template,
1794                 bool keyed)
1795 {
1796         struct caam_hash_alg *t_alg;
1797         struct ahash_alg *halg;
1798         struct crypto_alg *alg;
1799
1800         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1801         if (!t_alg) {
1802                 pr_err("failed to allocate t_alg\n");
1803                 return ERR_PTR(-ENOMEM);
1804         }
1805
1806         t_alg->ahash_alg = template->template_ahash;
1807         halg = &t_alg->ahash_alg;
1808         alg = &halg->halg.base;
1809
1810         if (keyed) {
1811                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1812                          template->hmac_name);
1813                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1814                          template->hmac_driver_name);
1815         } else {
1816                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1817                          template->name);
1818                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1819                          template->driver_name);
1820                 t_alg->ahash_alg.setkey = NULL;
1821         }
1822         alg->cra_module = THIS_MODULE;
1823         alg->cra_init = caam_hash_cra_init;
1824         alg->cra_exit = caam_hash_cra_exit;
1825         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1826         alg->cra_priority = CAAM_CRA_PRIORITY;
1827         alg->cra_blocksize = template->blocksize;
1828         alg->cra_alignmask = 0;
1829         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1830         alg->cra_type = &crypto_ahash_type;
1831
1832         t_alg->alg_type = template->alg_type;
1833
1834         return t_alg;
1835 }
1836
1837 static int __init caam_algapi_hash_init(void)
1838 {
1839         struct device_node *dev_node;
1840         struct platform_device *pdev;
1841         struct device *ctrldev;
1842         int i = 0, err = 0;
1843         struct caam_drv_private *priv;
1844         unsigned int md_limit = SHA512_DIGEST_SIZE;
1845         u32 cha_inst, cha_vid;
1846
1847         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1848         if (!dev_node) {
1849                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1850                 if (!dev_node)
1851                         return -ENODEV;
1852         }
1853
1854         pdev = of_find_device_by_node(dev_node);
1855         if (!pdev) {
1856                 of_node_put(dev_node);
1857                 return -ENODEV;
1858         }
1859
1860         ctrldev = &pdev->dev;
1861         priv = dev_get_drvdata(ctrldev);
1862         of_node_put(dev_node);
1863
1864         /*
1865          * If priv is NULL, it's probably because the caam driver wasn't
1866          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1867          */
1868         if (!priv)
1869                 return -ENODEV;
1870
1871         /*
1872          * Register crypto algorithms the device supports.  First, identify
1873          * presence and attributes of MD block.
1874          */
1875         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
1876         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1877
1878         /*
1879          * Skip registration of any hashing algorithms if MD block
1880          * is not present.
1881          */
1882         if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
1883                 return -ENODEV;
1884
1885         /* Limit digest size based on LP256 */
1886         if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
1887                 md_limit = SHA256_DIGEST_SIZE;
1888
1889         INIT_LIST_HEAD(&hash_list);
1890
1891         /* register crypto algorithms the device supports */
1892         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1893                 struct caam_hash_alg *t_alg;
1894                 struct caam_hash_template *alg = driver_hash + i;
1895
1896                 /* If MD size is not supported by device, skip registration */
1897                 if (alg->template_ahash.halg.digestsize > md_limit)
1898                         continue;
1899
1900                 /* register hmac version */
1901                 t_alg = caam_hash_alloc(alg, true);
1902                 if (IS_ERR(t_alg)) {
1903                         err = PTR_ERR(t_alg);
1904                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1905                         continue;
1906                 }
1907
1908                 err = crypto_register_ahash(&t_alg->ahash_alg);
1909                 if (err) {
1910                         pr_warn("%s alg registration failed: %d\n",
1911                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1912                                 err);
1913                         kfree(t_alg);
1914                 } else
1915                         list_add_tail(&t_alg->entry, &hash_list);
1916
1917                 /* register unkeyed version */
1918                 t_alg = caam_hash_alloc(alg, false);
1919                 if (IS_ERR(t_alg)) {
1920                         err = PTR_ERR(t_alg);
1921                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1922                         continue;
1923                 }
1924
1925                 err = crypto_register_ahash(&t_alg->ahash_alg);
1926                 if (err) {
1927                         pr_warn("%s alg registration failed: %d\n",
1928                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1929                                 err);
1930                         kfree(t_alg);
1931                 } else
1932                         list_add_tail(&t_alg->entry, &hash_list);
1933         }
1934
1935         return err;
1936 }
1937
1938 module_init(caam_algapi_hash_init);
1939 module_exit(caam_algapi_hash_exit);
1940
1941 MODULE_LICENSE("GPL");
1942 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
1943 MODULE_AUTHOR("Freescale Semiconductor - NMG");