drivers/crypto/inside-secure/safexcel_hash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <antoine.tenart@free-electrons.com>
   6  */
   7
   8 #include <crypto/aes.h>
   9 #include <crypto/hmac.h>
  10 #include <crypto/md5.h>
  11 #include <crypto/sha1.h>
  12 #include <crypto/sha2.h>
  13 #include <crypto/sha3.h>
  14 #include <crypto/skcipher.h>
  15 #include <crypto/sm3.h>
  16 #include <crypto/internal/cipher.h>
  17 #include <linux/device.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/dmapool.h>
  20
  21 #include "safexcel.h"
  22
  23 struct safexcel_ahash_ctx {
  24         struct safexcel_context base;
  25
  26         u32 alg;
  27         u8  key_sz;
  28         bool cbcmac;
  29         bool do_fallback;
  30         bool fb_init_done;
  31         bool fb_do_setkey;
  32
  33         struct crypto_aes_ctx *aes;
  34         struct crypto_ahash *fback;
  35         struct crypto_shash *shpre;
  36         struct shash_desc *shdesc;
  37 };
  38
  39 struct safexcel_ahash_req {
  40         bool last_req;
  41         bool finish;
  42         bool hmac;
  43         bool needs_inv;
  44         bool hmac_zlen;
  45         bool len_is_le;
  46         bool not_first;
  47         bool xcbcmac;
  48
  49         int nents;
  50         dma_addr_t result_dma;
  51
  52         u32 digest;
  53
  54         u8 state_sz;    /* expected state size, only set once */
  55         u8 block_sz;    /* block size, only set once */
  56         u8 digest_sz;   /* output digest size, only set once */
  57         __le32 state[SHA3_512_BLOCK_SIZE /
  58                      sizeof(__le32)] __aligned(sizeof(__le32));
  59
  60         u64 len;
  61         u64 processed;
  62
  63         u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  64         dma_addr_t cache_dma;
  65         unsigned int cache_sz;
  66
  67         u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  68 };
  69
  70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
  71 {
  72         return req->len - req->processed;
  73 }
  74
  75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
  76                                 u32 input_length, u32 result_length,
  77                                 bool cbcmac)
  78 {
  79         struct safexcel_token *token =
  80                 (struct safexcel_token *)cdesc->control_data.token;
  81
  82         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  83         token[0].packet_length = input_length;
  84         token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  85
  86         input_length &= 15;
  87         if (unlikely(cbcmac && input_length)) {
  88                 token[0].stat =  0;
  89                 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
  90                 token[1].packet_length = 16 - input_length;
  91                 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
  92                 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  93         } else {
  94                 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
  95                 eip197_noop_token(&token[1]);
  96         }
  97
  98         token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
  99         token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
 100                         EIP197_TOKEN_STAT_LAST_PACKET;
 101         token[2].packet_length = result_length;
 102         token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 103                                 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 104
 105         eip197_noop_token(&token[3]);
 106 }
 107
 108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
 109                                      struct safexcel_ahash_req *req,
 110                                      struct safexcel_command_desc *cdesc)
 111 {
 112         struct safexcel_crypto_priv *priv = ctx->base.priv;
 113         u64 count = 0;
 114
 115         cdesc->control_data.control0 = ctx->alg;
 116         cdesc->control_data.control1 = 0;
 117
 118         /*
 119          * Copy the input digest if needed, and setup the context
 120          * fields. Do this now as we need it to setup the first command
 121          * descriptor.
 122          */
 123         if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
 124                 if (req->xcbcmac)
 125                         memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
 126                 else
 127                         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 128
 129                 if (!req->finish && req->xcbcmac)
 130                         cdesc->control_data.control0 |=
 131                                 CONTEXT_CONTROL_DIGEST_XCM |
 132                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 133                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 134                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 135                                                      sizeof(u32));
 136                 else
 137                         cdesc->control_data.control0 |=
 138                                 CONTEXT_CONTROL_DIGEST_XCM |
 139                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 140                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 141                                                      sizeof(u32));
 142                 return;
 143         } else if (!req->processed) {
 144                 /* First - and possibly only - block of basic hash only */
 145                 if (req->finish)
 146                         cdesc->control_data.control0 |= req->digest |
 147                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 148                                 CONTEXT_CONTROL_RESTART_HASH  |
 149                                 /* ensure its not 0! */
 150                                 CONTEXT_CONTROL_SIZE(1);
 151                 else
 152                         cdesc->control_data.control0 |= req->digest |
 153                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 154                                 CONTEXT_CONTROL_RESTART_HASH   |
 155                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 156                                 /* ensure its not 0! */
 157                                 CONTEXT_CONTROL_SIZE(1);
 158                 return;
 159         }
 160
 161         /* Hash continuation or HMAC, setup (inner) digest from state */
 162         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 163
 164         if (req->finish) {
 165                 /* Compute digest count for hash/HMAC finish operations */
 166                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 167                     req->hmac_zlen || (req->processed != req->block_sz)) {
 168                         count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
 169
 170                         /* This is a hardware limitation, as the
 171                          * counter must fit into an u32. This represents
 172                          * a fairly big amount of input data, so we
 173                          * shouldn't see this.
 174                          */
 175                         if (unlikely(count & 0xffffffff00000000ULL)) {
 176                                 dev_warn(priv->dev,
 177                                          "Input data is too big\n");
 178                                 return;
 179                         }
 180                 }
 181
 182                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 183                     /* Special case: zero length HMAC */
 184                     req->hmac_zlen ||
 185                     /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
 186                     (req->processed != req->block_sz)) {
 187                         /* Basic hash continue operation, need digest + cnt */
 188                         cdesc->control_data.control0 |=
 189                                 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
 190                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 191                                 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 192                         /* For zero-len HMAC, don't finalize, already padded! */
 193                         if (req->hmac_zlen)
 194                                 cdesc->control_data.control0 |=
 195                                         CONTEXT_CONTROL_NO_FINISH_HASH;
 196                         cdesc->control_data.control1 |=
 197                                 CONTEXT_CONTROL_DIGEST_CNT;
 198                         ctx->base.ctxr->data[req->state_sz >> 2] =
 199                                 cpu_to_le32(count);
 200                         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 201
 202                         /* Clear zero-length HMAC flag for next operation! */
 203                         req->hmac_zlen = false;
 204                 } else { /* HMAC */
 205                         /* Need outer digest for HMAC finalization */
 206                         memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
 207                                &ctx->base.opad, req->state_sz);
 208
 209                         /* Single pass HMAC - no digest count */
 210                         cdesc->control_data.control0 |=
 211                                 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
 212                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 213                                 CONTEXT_CONTROL_DIGEST_HMAC;
 214                 }
 215         } else { /* Hash continuation, do not finish yet */
 216                 cdesc->control_data.control0 |=
 217                         CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
 218                         CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
 219                         CONTEXT_CONTROL_TYPE_HASH_OUT |
 220                         CONTEXT_CONTROL_NO_FINISH_HASH;
 221         }
 222 }
 223
 224 static int safexcel_ahash_enqueue(struct ahash_request *areq);
 225
 226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
 227                                       int ring,
 228                                       struct crypto_async_request *async,
 229                                       bool *should_complete, int *ret)
 230 {
 231         struct safexcel_result_desc *rdesc;
 232         struct ahash_request *areq = ahash_request_cast(async);
 233         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 234         struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq);
 235         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 236         u64 cache_len;
 237
 238         *ret = 0;
 239
 240         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 241         if (IS_ERR(rdesc)) {
 242                 dev_err(priv->dev,
 243                         "hash: result: could not retrieve the result descriptor\n");
 244                 *ret = PTR_ERR(rdesc);
 245         } else {
 246                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 247         }
 248
 249         safexcel_complete(priv, ring);
 250
 251         if (sreq->nents) {
 252                 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
 253                 sreq->nents = 0;
 254         }
 255
 256         if (sreq->result_dma) {
 257                 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
 258                                  DMA_FROM_DEVICE);
 259                 sreq->result_dma = 0;
 260         }
 261
 262         if (sreq->cache_dma) {
 263                 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
 264                                  DMA_TO_DEVICE);
 265                 sreq->cache_dma = 0;
 266                 sreq->cache_sz = 0;
 267         }
 268
 269         if (sreq->finish) {
 270                 if (sreq->hmac &&
 271                     (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
 272                         /* Faking HMAC using hash - need to do outer hash */
 273                         memcpy(sreq->cache, sreq->state,
 274                                crypto_ahash_digestsize(ahash));
 275
 276                         memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
 277
 278                         sreq->len = sreq->block_sz +
 279                                     crypto_ahash_digestsize(ahash);
 280                         sreq->processed = sreq->block_sz;
 281                         sreq->hmac = 0;
 282
 283                         if (priv->flags & EIP197_TRC_CACHE)
 284                                 ctx->base.needs_inv = true;
 285                         areq->nbytes = 0;
 286                         safexcel_ahash_enqueue(areq);
 287
 288                         *should_complete = false; /* Not done yet */
 289                         return 1;
 290                 }
 291
 292                 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 293                              ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
 294                         /* Undo final XOR with 0xffffffff ...*/
 295                         *(__le32 *)areq->result = ~sreq->state[0];
 296                 } else {
 297                         memcpy(areq->result, sreq->state,
 298                                crypto_ahash_digestsize(ahash));
 299                 }
 300         }
 301
 302         cache_len = safexcel_queued_len(sreq);
 303         if (cache_len)
 304                 memcpy(sreq->cache, sreq->cache_next, cache_len);
 305
 306         *should_complete = true;
 307
 308         return 1;
 309 }
 310
 311 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 312                                    int *commands, int *results)
 313 {
 314         struct ahash_request *areq = ahash_request_cast(async);
 315         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 316         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 317         struct safexcel_crypto_priv *priv = ctx->base.priv;
 318         struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 319         struct safexcel_result_desc *rdesc;
 320         struct scatterlist *sg;
 321         struct safexcel_token *dmmy;
 322         int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
 323         u64 queued, len;
 324
 325         queued = safexcel_queued_len(req);
 326         if (queued <= HASH_CACHE_SIZE)
 327                 cache_len = queued;
 328         else
 329                 cache_len = queued - areq->nbytes;
 330
 331         if (!req->finish && !req->last_req) {
 332                 /* If this is not the last request and the queued data does not
 333                  * fit into full cache blocks, cache it for the next send call.
 334                  */
 335                 extra = queued & (HASH_CACHE_SIZE - 1);
 336
 337                 /* If this is not the last request and the queued data
 338                  * is a multiple of a block, cache the last one for now.
 339                  */
 340                 if (!extra)
 341                         extra = HASH_CACHE_SIZE;
 342
 343                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 344                                    req->cache_next, extra,
 345                                    areq->nbytes - extra);
 346
 347                 queued -= extra;
 348
 349                 if (!queued) {
 350                         *commands = 0;
 351                         *results = 0;
 352                         return 0;
 353                 }
 354
 355                 extra = 0;
 356         }
 357
 358         if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
 359                 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
 360                         /*
 361                          * Cache contains less than 1 full block, complete.
 362                          */
 363                         extra = AES_BLOCK_SIZE - cache_len;
 364                         if (queued > cache_len) {
 365                                 /* More data follows: borrow bytes */
 366                                 u64 tmp = queued - cache_len;
 367
 368                                 skip = min_t(u64, tmp, extra);
 369                                 sg_pcopy_to_buffer(areq->src,
 370                                         sg_nents(areq->src),
 371                                         req->cache + cache_len,
 372                                         skip, 0);
 373                         }
 374                         extra -= skip;
 375                         memset(req->cache + cache_len + skip, 0, extra);
 376                         if (!ctx->cbcmac && extra) {
 377                                 // 10- padding for XCBCMAC & CMAC
 378                                 req->cache[cache_len + skip] = 0x80;
 379                                 // HW will use K2 iso K3 - compensate!
 380                                 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
 381                                         u32 *cache = (void *)req->cache;
 382                                         u32 *ipad = ctx->base.ipad.word;
 383                                         u32 x;
 384
 385                                         x = ipad[i] ^ ipad[i + 4];
 386                                         cache[i] ^= swab32(x);
 387                                 }
 388                         }
 389                         cache_len = AES_BLOCK_SIZE;
 390                         queued = queued + extra;
 391                 }
 392
 393                 /* XCBC continue: XOR previous result into 1st word */
 394                 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
 395         }
 396
 397         len = queued;
 398         /* Add a command descriptor for the cached data, if any */
 399         if (cache_len) {
 400                 req->cache_dma = dma_map_single(priv->dev, req->cache,
 401                                                 cache_len, DMA_TO_DEVICE);
 402                 if (dma_mapping_error(priv->dev, req->cache_dma))
 403                         return -EINVAL;
 404
 405                 req->cache_sz = cache_len;
 406                 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
 407                                                  (cache_len == len),
 408                                                  req->cache_dma, cache_len,
 409                                                  len, ctx->base.ctxr_dma,
 410                                                  &dmmy);
 411                 if (IS_ERR(first_cdesc)) {
 412                         ret = PTR_ERR(first_cdesc);
 413                         goto unmap_cache;
 414                 }
 415                 n_cdesc++;
 416
 417                 queued -= cache_len;
 418                 if (!queued)
 419                         goto send_command;
 420         }
 421
 422         /* Now handle the current ahash request buffer(s) */
 423         req->nents = dma_map_sg(priv->dev, areq->src,
 424                                 sg_nents_for_len(areq->src,
 425                                                  areq->nbytes),
 426                                 DMA_TO_DEVICE);
 427         if (!req->nents) {
 428                 ret = -ENOMEM;
 429                 goto cdesc_rollback;
 430         }
 431
 432         for_each_sg(areq->src, sg, req->nents, i) {
 433                 int sglen = sg_dma_len(sg);
 434
 435                 if (unlikely(sglen <= skip)) {
 436                         skip -= sglen;
 437                         continue;
 438                 }
 439
 440                 /* Do not overflow the request */
 441                 if ((queued + skip) <= sglen)
 442                         sglen = queued;
 443                 else
 444                         sglen -= skip;
 445
 446                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 447                                            !(queued - sglen),
 448                                            sg_dma_address(sg) + skip, sglen,
 449                                            len, ctx->base.ctxr_dma, &dmmy);
 450                 if (IS_ERR(cdesc)) {
 451                         ret = PTR_ERR(cdesc);
 452                         goto unmap_sg;
 453                 }
 454
 455                 if (!n_cdesc)
 456                         first_cdesc = cdesc;
 457                 n_cdesc++;
 458
 459                 queued -= sglen;
 460                 if (!queued)
 461                         break;
 462                 skip = 0;
 463         }
 464
 465 send_command:
 466         /* Setup the context options */
 467         safexcel_context_control(ctx, req, first_cdesc);
 468
 469         /* Add the token */
 470         safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
 471
 472         req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
 473                                          DMA_FROM_DEVICE);
 474         if (dma_mapping_error(priv->dev, req->result_dma)) {
 475                 ret = -EINVAL;
 476                 goto unmap_sg;
 477         }
 478
 479         /* Add a result descriptor */
 480         rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
 481                                    req->digest_sz);
 482         if (IS_ERR(rdesc)) {
 483                 ret = PTR_ERR(rdesc);
 484                 goto unmap_result;
 485         }
 486
 487         safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
 488
 489         req->processed += len - extra;
 490
 491         *commands = n_cdesc;
 492         *results = 1;
 493         return 0;
 494
 495 unmap_result:
 496         dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
 497                          DMA_FROM_DEVICE);
 498 unmap_sg:
 499         if (req->nents) {
 500                 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
 501                 req->nents = 0;
 502         }
 503 cdesc_rollback:
 504         for (i = 0; i < n_cdesc; i++)
 505                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 506 unmap_cache:
 507         if (req->cache_dma) {
 508                 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
 509                                  DMA_TO_DEVICE);
 510                 req->cache_dma = 0;
 511                 req->cache_sz = 0;
 512         }
 513
 514         return ret;
 515 }
 516
 517 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 518                                       int ring,
 519                                       struct crypto_async_request *async,
 520                                       bool *should_complete, int *ret)
 521 {
 522         struct safexcel_result_desc *rdesc;
 523         struct ahash_request *areq = ahash_request_cast(async);
 524         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 525         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 526         int enq_ret;
 527
 528         *ret = 0;
 529
 530         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 531         if (IS_ERR(rdesc)) {
 532                 dev_err(priv->dev,
 533                         "hash: invalidate: could not retrieve the result descriptor\n");
 534                 *ret = PTR_ERR(rdesc);
 535         } else {
 536                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 537         }
 538
 539         safexcel_complete(priv, ring);
 540
 541         if (ctx->base.exit_inv) {
 542                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 543                               ctx->base.ctxr_dma);
 544
 545                 *should_complete = true;
 546                 return 1;
 547         }
 548
 549         ring = safexcel_select_ring(priv);
 550         ctx->base.ring = ring;
 551
 552         spin_lock_bh(&priv->ring[ring].queue_lock);
 553         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
 554         spin_unlock_bh(&priv->ring[ring].queue_lock);
 555
 556         if (enq_ret != -EINPROGRESS)
 557                 *ret = enq_ret;
 558
 559         queue_work(priv->ring[ring].workqueue,
 560                    &priv->ring[ring].work_data.work);
 561
 562         *should_complete = false;
 563
 564         return 1;
 565 }
 566
 567 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 568                                   struct crypto_async_request *async,
 569                                   bool *should_complete, int *ret)
 570 {
 571         struct ahash_request *areq = ahash_request_cast(async);
 572         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 573         int err;
 574
 575         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
 576
 577         if (req->needs_inv) {
 578                 req->needs_inv = false;
 579                 err = safexcel_handle_inv_result(priv, ring, async,
 580                                                  should_complete, ret);
 581         } else {
 582                 err = safexcel_handle_req_result(priv, ring, async,
 583                                                  should_complete, ret);
 584         }
 585
 586         return err;
 587 }
 588
 589 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 590                                    int ring, int *commands, int *results)
 591 {
 592         struct ahash_request *areq = ahash_request_cast(async);
 593         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 594         int ret;
 595
 596         ret = safexcel_invalidate_cache(async, ctx->base.priv,
 597                                         ctx->base.ctxr_dma, ring);
 598         if (unlikely(ret))
 599                 return ret;
 600
 601         *commands = 1;
 602         *results = 1;
 603
 604         return 0;
 605 }
 606
 607 static int safexcel_ahash_send(struct crypto_async_request *async,
 608                                int ring, int *commands, int *results)
 609 {
 610         struct ahash_request *areq = ahash_request_cast(async);
 611         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 612         int ret;
 613
 614         if (req->needs_inv)
 615                 ret = safexcel_ahash_send_inv(async, ring, commands, results);
 616         else
 617                 ret = safexcel_ahash_send_req(async, ring, commands, results);
 618
 619         return ret;
 620 }
 621
 622 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 623 {
 624         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 625         struct safexcel_crypto_priv *priv = ctx->base.priv;
 626         EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
 627         struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req);
 628         DECLARE_CRYPTO_WAIT(result);
 629         int ring = ctx->base.ring;
 630         int err;
 631
 632         memset(req, 0, EIP197_AHASH_REQ_SIZE);
 633
 634         /* create invalidation request */
 635         init_completion(&result.completion);
 636         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 637                                    crypto_req_done, &result);
 638
 639         ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
 640         ctx = crypto_tfm_ctx(req->base.tfm);
 641         ctx->base.exit_inv = true;
 642         rctx->needs_inv = true;
 643
 644         spin_lock_bh(&priv->ring[ring].queue_lock);
 645         crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 646         spin_unlock_bh(&priv->ring[ring].queue_lock);
 647
 648         queue_work(priv->ring[ring].workqueue,
 649                    &priv->ring[ring].work_data.work);
 650
 651         err = crypto_wait_req(-EINPROGRESS, &result);
 652
 653         if (err) {
 654                 dev_warn(priv->dev, "hash: completion error (%d)\n", err);
 655                 return err;
 656         }
 657
 658         return 0;
 659 }
 660
 661 /* safexcel_ahash_cache: cache data until at least one request can be sent to
 662  * the engine, aka. when there is at least 1 block size in the pipe.
 663  */
 664 static int safexcel_ahash_cache(struct ahash_request *areq)
 665 {
 666         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 667         u64 cache_len;
 668
 669         /* cache_len: everything accepted by the driver but not sent yet,
 670          * tot sz handled by update() - last req sz - tot sz handled by send()
 671          */
 672         cache_len = safexcel_queued_len(req);
 673
 674         /*
 675          * In case there isn't enough bytes to proceed (less than a
 676          * block size), cache the data until we have enough.
 677          */
 678         if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
 679                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 680                                    req->cache + cache_len,
 681                                    areq->nbytes, 0);
 682                 return 0;
 683         }
 684
 685         /* We couldn't cache all the data */
 686         return -E2BIG;
 687 }
 688
 689 static int safexcel_ahash_enqueue(struct ahash_request *areq)
 690 {
 691         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 692         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 693         struct safexcel_crypto_priv *priv = ctx->base.priv;
 694         int ret, ring;
 695
 696         req->needs_inv = false;
 697
 698         if (ctx->base.ctxr) {
 699                 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
 700                      /* invalidate for *any* non-XCBC continuation */
 701                    ((req->not_first && !req->xcbcmac) ||
 702                      /* invalidate if (i)digest changed */
 703                      memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
 704                      /* invalidate for HMAC finish with odigest changed */
 705                      (req->finish && req->hmac &&
 706                       memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
 707                              &ctx->base.opad, req->state_sz))))
 708                         /*
 709                          * We're still setting needs_inv here, even though it is
 710                          * cleared right away, because the needs_inv flag can be
 711                          * set in other functions and we want to keep the same
 712                          * logic.
 713                          */
 714                         ctx->base.needs_inv = true;
 715
 716                 if (ctx->base.needs_inv) {
 717                         ctx->base.needs_inv = false;
 718                         req->needs_inv = true;
 719                 }
 720         } else {
 721                 ctx->base.ring = safexcel_select_ring(priv);
 722                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 723                                                  EIP197_GFP_FLAGS(areq->base),
 724                                                  &ctx->base.ctxr_dma);
 725                 if (!ctx->base.ctxr)
 726                         return -ENOMEM;
 727         }
 728         req->not_first = true;
 729
 730         ring = ctx->base.ring;
 731
 732         spin_lock_bh(&priv->ring[ring].queue_lock);
 733         ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 734         spin_unlock_bh(&priv->ring[ring].queue_lock);
 735
 736         queue_work(priv->ring[ring].workqueue,
 737                    &priv->ring[ring].work_data.work);
 738
 739         return ret;
 740 }
 741
 742 static int safexcel_ahash_update(struct ahash_request *areq)
 743 {
 744         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 745         int ret;
 746
 747         /* If the request is 0 length, do nothing */
 748         if (!areq->nbytes)
 749                 return 0;
 750
 751         /* Add request to the cache if it fits */
 752         ret = safexcel_ahash_cache(areq);
 753
 754         /* Update total request length */
 755         req->len += areq->nbytes;
 756
 757         /* If not all data could fit into the cache, go process the excess.
 758          * Also go process immediately for an HMAC IV precompute, which
 759          * will never be finished at all, but needs to be processed anyway.
 760          */
 761         if ((ret && !req->finish) || req->last_req)
 762                 return safexcel_ahash_enqueue(areq);
 763
 764         return 0;
 765 }
 766
 767 static int safexcel_ahash_final(struct ahash_request *areq)
 768 {
 769         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 770         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 771
 772         req->finish = true;
 773
 774         if (unlikely(!req->len && !areq->nbytes)) {
 775                 /*
 776                  * If we have an overall 0 length *hash* request:
 777                  * The HW cannot do 0 length hash, so we provide the correct
 778                  * result directly here.
 779                  */
 780                 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
 781                         memcpy(areq->result, md5_zero_message_hash,
 782                                MD5_DIGEST_SIZE);
 783                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
 784                         memcpy(areq->result, sha1_zero_message_hash,
 785                                SHA1_DIGEST_SIZE);
 786                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
 787                         memcpy(areq->result, sha224_zero_message_hash,
 788                                SHA224_DIGEST_SIZE);
 789                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
 790                         memcpy(areq->result, sha256_zero_message_hash,
 791                                SHA256_DIGEST_SIZE);
 792                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
 793                         memcpy(areq->result, sha384_zero_message_hash,
 794                                SHA384_DIGEST_SIZE);
 795                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
 796                         memcpy(areq->result, sha512_zero_message_hash,
 797                                SHA512_DIGEST_SIZE);
 798                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
 799                         memcpy(areq->result,
 800                                EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
 801                 }
 802
 803                 return 0;
 804         } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 805                             ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
 806                             req->len == sizeof(u32) && !areq->nbytes)) {
 807                 /* Zero length CRC32 */
 808                 memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
 809                 return 0;
 810         } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
 811                             !areq->nbytes)) {
 812                 /* Zero length CBC MAC */
 813                 memset(areq->result, 0, AES_BLOCK_SIZE);
 814                 return 0;
 815         } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
 816                             !areq->nbytes)) {
 817                 /* Zero length (X)CBC/CMAC */
 818                 int i;
 819
 820                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
 821                         u32 *result = (void *)areq->result;
 822
 823                         /* K3 */
 824                         result[i] = swab32(ctx->base.ipad.word[i + 4]);
 825                 }
 826                 areq->result[0] ^= 0x80;                        // 10- padding
 827                 aes_encrypt(ctx->aes, areq->result, areq->result);
 828                 return 0;
 829         } else if (unlikely(req->hmac &&
 830                             (req->len == req->block_sz) &&
 831                             !areq->nbytes)) {
 832                 /*
 833                  * If we have an overall 0 length *HMAC* request:
 834                  * For HMAC, we need to finalize the inner digest
 835                  * and then perform the outer hash.
 836                  */
 837
 838                 /* generate pad block in the cache */
 839                 /* start with a hash block of all zeroes */
 840                 memset(req->cache, 0, req->block_sz);
 841                 /* set the first byte to 0x80 to 'append a 1 bit' */
 842                 req->cache[0] = 0x80;
 843                 /* add the length in bits in the last 2 bytes */
 844                 if (req->len_is_le) {
 845                         /* Little endian length word (e.g. MD5) */
 846                         req->cache[req->block_sz-8] = (req->block_sz << 3) &
 847                                                       255;
 848                         req->cache[req->block_sz-7] = (req->block_sz >> 5);
 849                 } else {
 850                         /* Big endian length word (e.g. any SHA) */
 851                         req->cache[req->block_sz-2] = (req->block_sz >> 5);
 852                         req->cache[req->block_sz-1] = (req->block_sz << 3) &
 853                                                       255;
 854                 }
 855
 856                 req->len += req->block_sz; /* plus 1 hash block */
 857
 858                 /* Set special zero-length HMAC flag */
 859                 req->hmac_zlen = true;
 860
 861                 /* Finalize HMAC */
 862                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 863         } else if (req->hmac) {
 864                 /* Finalize HMAC */
 865                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 866         }
 867
 868         return safexcel_ahash_enqueue(areq);
 869 }
 870
 871 static int safexcel_ahash_finup(struct ahash_request *areq)
 872 {
 873         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 874
 875         req->finish = true;
 876
 877         safexcel_ahash_update(areq);
 878         return safexcel_ahash_final(areq);
 879 }
 880
 881 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 882 {
 883         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 884         struct safexcel_ahash_export_state *export = out;
 885
 886         export->len = req->len;
 887         export->processed = req->processed;
 888
 889         export->digest = req->digest;
 890
 891         memcpy(export->state, req->state, req->state_sz);
 892         memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
 893
 894         return 0;
 895 }
 896
 897 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 898 {
 899         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 900         const struct safexcel_ahash_export_state *export = in;
 901         int ret;
 902
 903         ret = crypto_ahash_init(areq);
 904         if (ret)
 905                 return ret;
 906
 907         req->len = export->len;
 908         req->processed = export->processed;
 909
 910         req->digest = export->digest;
 911
 912         memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
 913         memcpy(req->state, export->state, req->state_sz);
 914
 915         return 0;
 916 }
 917
 918 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
 919 {
 920         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 921         struct safexcel_alg_template *tmpl =
 922                 container_of(__crypto_ahash_alg(tfm->__crt_alg),
 923                              struct safexcel_alg_template, alg.ahash);
 924
 925         ctx->base.priv = tmpl->priv;
 926         ctx->base.send = safexcel_ahash_send;
 927         ctx->base.handle_result = safexcel_handle_result;
 928         ctx->fb_do_setkey = false;
 929
 930         crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
 931                                      sizeof(struct safexcel_ahash_req));
 932         return 0;
 933 }
 934
 935 static int safexcel_sha1_init(struct ahash_request *areq)
 936 {
 937         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 938         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 939
 940         memset(req, 0, sizeof(*req));
 941
 942         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
 943         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 944         req->state_sz = SHA1_DIGEST_SIZE;
 945         req->digest_sz = SHA1_DIGEST_SIZE;
 946         req->block_sz = SHA1_BLOCK_SIZE;
 947
 948         return 0;
 949 }
 950
 951 static int safexcel_sha1_digest(struct ahash_request *areq)
 952 {
 953         int ret = safexcel_sha1_init(areq);
 954
 955         if (ret)
 956                 return ret;
 957
 958         return safexcel_ahash_finup(areq);
 959 }
 960
 961 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 962 {
 963         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 964         struct safexcel_crypto_priv *priv = ctx->base.priv;
 965         int ret;
 966
 967         /* context not allocated, skip invalidation */
 968         if (!ctx->base.ctxr)
 969                 return;
 970
 971         if (priv->flags & EIP197_TRC_CACHE) {
 972                 ret = safexcel_ahash_exit_inv(tfm);
 973                 if (ret)
 974                         dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
 975         } else {
 976                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 977                               ctx->base.ctxr_dma);
 978         }
 979 }
 980
 981 struct safexcel_alg_template safexcel_alg_sha1 = {
 982         .type = SAFEXCEL_ALG_TYPE_AHASH,
 983         .algo_mask = SAFEXCEL_ALG_SHA1,
 984         .alg.ahash = {
 985                 .init = safexcel_sha1_init,
 986                 .update = safexcel_ahash_update,
 987                 .final = safexcel_ahash_final,
 988                 .finup = safexcel_ahash_finup,
 989                 .digest = safexcel_sha1_digest,
 990                 .export = safexcel_ahash_export,
 991                 .import = safexcel_ahash_import,
 992                 .halg = {
 993                         .digestsize = SHA1_DIGEST_SIZE,
 994                         .statesize = sizeof(struct safexcel_ahash_export_state),
 995                         .base = {
 996                                 .cra_name = "sha1",
 997                                 .cra_driver_name = "safexcel-sha1",
 998                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
 999                                 .cra_flags = CRYPTO_ALG_ASYNC |
1000                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1001                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1002                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1003                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1004                                 .cra_init = safexcel_ahash_cra_init,
1005                                 .cra_exit = safexcel_ahash_cra_exit,
1006                                 .cra_module = THIS_MODULE,
1007                         },
1008                 },
1009         },
1010 };
1011
1012 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1013 {
1014         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1015         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1016
1017         memset(req, 0, sizeof(*req));
1018
1019         /* Start from ipad precompute */
1020         memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1021         /* Already processed the key^ipad part now! */
1022         req->len        = SHA1_BLOCK_SIZE;
1023         req->processed  = SHA1_BLOCK_SIZE;
1024
1025         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1026         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1027         req->state_sz = SHA1_DIGEST_SIZE;
1028         req->digest_sz = SHA1_DIGEST_SIZE;
1029         req->block_sz = SHA1_BLOCK_SIZE;
1030         req->hmac = true;
1031
1032         return 0;
1033 }
1034
1035 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1036 {
1037         int ret = safexcel_hmac_sha1_init(areq);
1038
1039         if (ret)
1040                 return ret;
1041
1042         return safexcel_ahash_finup(areq);
1043 }
1044
1045 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1046                                   unsigned int blocksize, const u8 *key,
1047                                   unsigned int keylen, u8 *ipad, u8 *opad)
1048 {
1049         DECLARE_CRYPTO_WAIT(result);
1050         struct scatterlist sg;
1051         int ret, i;
1052         u8 *keydup;
1053
1054         if (keylen <= blocksize) {
1055                 memcpy(ipad, key, keylen);
1056         } else {
1057                 keydup = kmemdup(key, keylen, GFP_KERNEL);
1058                 if (!keydup)
1059                         return -ENOMEM;
1060
1061                 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1062                                            crypto_req_done, &result);
1063                 sg_init_one(&sg, keydup, keylen);
1064                 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1065
1066                 ret = crypto_ahash_digest(areq);
1067                 ret = crypto_wait_req(ret, &result);
1068
1069                 /* Avoid leaking */
1070                 kfree_sensitive(keydup);
1071
1072                 if (ret)
1073                         return ret;
1074
1075                 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1076         }
1077
1078         memset(ipad + keylen, 0, blocksize - keylen);
1079         memcpy(opad, ipad, blocksize);
1080
1081         for (i = 0; i < blocksize; i++) {
1082                 ipad[i] ^= HMAC_IPAD_VALUE;
1083                 opad[i] ^= HMAC_OPAD_VALUE;
1084         }
1085
1086         return 0;
1087 }
1088
1089 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1090                                  unsigned int blocksize, u8 *pad, void *state)
1091 {
1092         struct safexcel_ahash_req *req;
1093         DECLARE_CRYPTO_WAIT(result);
1094         struct scatterlist sg;
1095         int ret;
1096
1097         ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1098                                    crypto_req_done, &result);
1099         sg_init_one(&sg, pad, blocksize);
1100         ahash_request_set_crypt(areq, &sg, pad, blocksize);
1101
1102         ret = crypto_ahash_init(areq);
1103         if (ret)
1104                 return ret;
1105
1106         req = ahash_request_ctx_dma(areq);
1107         req->hmac = true;
1108         req->last_req = true;
1109
1110         ret = crypto_ahash_update(areq);
1111         ret = crypto_wait_req(ret, &result);
1112
1113         return ret ?: crypto_ahash_export(areq, state);
1114 }
1115
1116 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1117                                   unsigned int keylen,
1118                                   void *istate, void *ostate)
1119 {
1120         struct ahash_request *areq;
1121         struct crypto_ahash *tfm;
1122         unsigned int blocksize;
1123         u8 *ipad, *opad;
1124         int ret;
1125
1126         tfm = crypto_alloc_ahash(alg, 0, 0);
1127         if (IS_ERR(tfm))
1128                 return PTR_ERR(tfm);
1129
1130         areq = ahash_request_alloc(tfm, GFP_KERNEL);
1131         if (!areq) {
1132                 ret = -ENOMEM;
1133                 goto free_ahash;
1134         }
1135
1136         crypto_ahash_clear_flags(tfm, ~0);
1137         blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1138
1139         ipad = kcalloc(2, blocksize, GFP_KERNEL);
1140         if (!ipad) {
1141                 ret = -ENOMEM;
1142                 goto free_request;
1143         }
1144
1145         opad = ipad + blocksize;
1146
1147         ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1148         if (ret)
1149                 goto free_ipad;
1150
1151         ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1152         if (ret)
1153                 goto free_ipad;
1154
1155         ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1156
1157 free_ipad:
1158         kfree(ipad);
1159 free_request:
1160         ahash_request_free(areq);
1161 free_ahash:
1162         crypto_free_ahash(tfm);
1163
1164         return ret;
1165 }
1166
1167 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1168                          unsigned int keylen, const char *alg,
1169                          unsigned int state_sz)
1170 {
1171         struct safexcel_crypto_priv *priv = base->priv;
1172         struct safexcel_ahash_export_state istate, ostate;
1173         int ret;
1174
1175         ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1176         if (ret)
1177                 return ret;
1178
1179         if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1180             (memcmp(&base->ipad, istate.state, state_sz) ||
1181              memcmp(&base->opad, ostate.state, state_sz)))
1182                 base->needs_inv = true;
1183
1184         memcpy(&base->ipad, &istate.state, state_sz);
1185         memcpy(&base->opad, &ostate.state, state_sz);
1186
1187         return 0;
1188 }
1189
1190 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1191                                     unsigned int keylen, const char *alg,
1192                                     unsigned int state_sz)
1193 {
1194         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1195
1196         return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1197 }
1198
1199 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1200                                      unsigned int keylen)
1201 {
1202         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1203                                         SHA1_DIGEST_SIZE);
1204 }
1205
1206 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1207         .type = SAFEXCEL_ALG_TYPE_AHASH,
1208         .algo_mask = SAFEXCEL_ALG_SHA1,
1209         .alg.ahash = {
1210                 .init = safexcel_hmac_sha1_init,
1211                 .update = safexcel_ahash_update,
1212                 .final = safexcel_ahash_final,
1213                 .finup = safexcel_ahash_finup,
1214                 .digest = safexcel_hmac_sha1_digest,
1215                 .setkey = safexcel_hmac_sha1_setkey,
1216                 .export = safexcel_ahash_export,
1217                 .import = safexcel_ahash_import,
1218                 .halg = {
1219                         .digestsize = SHA1_DIGEST_SIZE,
1220                         .statesize = sizeof(struct safexcel_ahash_export_state),
1221                         .base = {
1222                                 .cra_name = "hmac(sha1)",
1223                                 .cra_driver_name = "safexcel-hmac-sha1",
1224                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1225                                 .cra_flags = CRYPTO_ALG_ASYNC |
1226                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1227                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1228                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1229                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1230                                 .cra_init = safexcel_ahash_cra_init,
1231                                 .cra_exit = safexcel_ahash_cra_exit,
1232                                 .cra_module = THIS_MODULE,
1233                         },
1234                 },
1235         },
1236 };
1237
1238 static int safexcel_sha256_init(struct ahash_request *areq)
1239 {
1240         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1241         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1242
1243         memset(req, 0, sizeof(*req));
1244
1245         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1246         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1247         req->state_sz = SHA256_DIGEST_SIZE;
1248         req->digest_sz = SHA256_DIGEST_SIZE;
1249         req->block_sz = SHA256_BLOCK_SIZE;
1250
1251         return 0;
1252 }
1253
1254 static int safexcel_sha256_digest(struct ahash_request *areq)
1255 {
1256         int ret = safexcel_sha256_init(areq);
1257
1258         if (ret)
1259                 return ret;
1260
1261         return safexcel_ahash_finup(areq);
1262 }
1263
1264 struct safexcel_alg_template safexcel_alg_sha256 = {
1265         .type = SAFEXCEL_ALG_TYPE_AHASH,
1266         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1267         .alg.ahash = {
1268                 .init = safexcel_sha256_init,
1269                 .update = safexcel_ahash_update,
1270                 .final = safexcel_ahash_final,
1271                 .finup = safexcel_ahash_finup,
1272                 .digest = safexcel_sha256_digest,
1273                 .export = safexcel_ahash_export,
1274                 .import = safexcel_ahash_import,
1275                 .halg = {
1276                         .digestsize = SHA256_DIGEST_SIZE,
1277                         .statesize = sizeof(struct safexcel_ahash_export_state),
1278                         .base = {
1279                                 .cra_name = "sha256",
1280                                 .cra_driver_name = "safexcel-sha256",
1281                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1282                                 .cra_flags = CRYPTO_ALG_ASYNC |
1283                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1284                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1285                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1286                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1287                                 .cra_init = safexcel_ahash_cra_init,
1288                                 .cra_exit = safexcel_ahash_cra_exit,
1289                                 .cra_module = THIS_MODULE,
1290                         },
1291                 },
1292         },
1293 };
1294
1295 static int safexcel_sha224_init(struct ahash_request *areq)
1296 {
1297         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1298         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1299
1300         memset(req, 0, sizeof(*req));
1301
1302         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1303         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1304         req->state_sz = SHA256_DIGEST_SIZE;
1305         req->digest_sz = SHA256_DIGEST_SIZE;
1306         req->block_sz = SHA256_BLOCK_SIZE;
1307
1308         return 0;
1309 }
1310
1311 static int safexcel_sha224_digest(struct ahash_request *areq)
1312 {
1313         int ret = safexcel_sha224_init(areq);
1314
1315         if (ret)
1316                 return ret;
1317
1318         return safexcel_ahash_finup(areq);
1319 }
1320
1321 struct safexcel_alg_template safexcel_alg_sha224 = {
1322         .type = SAFEXCEL_ALG_TYPE_AHASH,
1323         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1324         .alg.ahash = {
1325                 .init = safexcel_sha224_init,
1326                 .update = safexcel_ahash_update,
1327                 .final = safexcel_ahash_final,
1328                 .finup = safexcel_ahash_finup,
1329                 .digest = safexcel_sha224_digest,
1330                 .export = safexcel_ahash_export,
1331                 .import = safexcel_ahash_import,
1332                 .halg = {
1333                         .digestsize = SHA224_DIGEST_SIZE,
1334                         .statesize = sizeof(struct safexcel_ahash_export_state),
1335                         .base = {
1336                                 .cra_name = "sha224",
1337                                 .cra_driver_name = "safexcel-sha224",
1338                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1339                                 .cra_flags = CRYPTO_ALG_ASYNC |
1340                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1341                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1342                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1343                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1344                                 .cra_init = safexcel_ahash_cra_init,
1345                                 .cra_exit = safexcel_ahash_cra_exit,
1346                                 .cra_module = THIS_MODULE,
1347                         },
1348                 },
1349         },
1350 };
1351
1352 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1353                                        unsigned int keylen)
1354 {
1355         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1356                                         SHA256_DIGEST_SIZE);
1357 }
1358
1359 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1360 {
1361         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1362         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1363
1364         memset(req, 0, sizeof(*req));
1365
1366         /* Start from ipad precompute */
1367         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1368         /* Already processed the key^ipad part now! */
1369         req->len        = SHA256_BLOCK_SIZE;
1370         req->processed  = SHA256_BLOCK_SIZE;
1371
1372         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1373         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1374         req->state_sz = SHA256_DIGEST_SIZE;
1375         req->digest_sz = SHA256_DIGEST_SIZE;
1376         req->block_sz = SHA256_BLOCK_SIZE;
1377         req->hmac = true;
1378
1379         return 0;
1380 }
1381
1382 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1383 {
1384         int ret = safexcel_hmac_sha224_init(areq);
1385
1386         if (ret)
1387                 return ret;
1388
1389         return safexcel_ahash_finup(areq);
1390 }
1391
1392 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1393         .type = SAFEXCEL_ALG_TYPE_AHASH,
1394         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1395         .alg.ahash = {
1396                 .init = safexcel_hmac_sha224_init,
1397                 .update = safexcel_ahash_update,
1398                 .final = safexcel_ahash_final,
1399                 .finup = safexcel_ahash_finup,
1400                 .digest = safexcel_hmac_sha224_digest,
1401                 .setkey = safexcel_hmac_sha224_setkey,
1402                 .export = safexcel_ahash_export,
1403                 .import = safexcel_ahash_import,
1404                 .halg = {
1405                         .digestsize = SHA224_DIGEST_SIZE,
1406                         .statesize = sizeof(struct safexcel_ahash_export_state),
1407                         .base = {
1408                                 .cra_name = "hmac(sha224)",
1409                                 .cra_driver_name = "safexcel-hmac-sha224",
1410                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1411                                 .cra_flags = CRYPTO_ALG_ASYNC |
1412                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1413                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1414                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1415                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1416                                 .cra_init = safexcel_ahash_cra_init,
1417                                 .cra_exit = safexcel_ahash_cra_exit,
1418                                 .cra_module = THIS_MODULE,
1419                         },
1420                 },
1421         },
1422 };
1423
1424 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1425                                      unsigned int keylen)
1426 {
1427         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1428                                         SHA256_DIGEST_SIZE);
1429 }
1430
1431 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1432 {
1433         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1434         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1435
1436         memset(req, 0, sizeof(*req));
1437
1438         /* Start from ipad precompute */
1439         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1440         /* Already processed the key^ipad part now! */
1441         req->len        = SHA256_BLOCK_SIZE;
1442         req->processed  = SHA256_BLOCK_SIZE;
1443
1444         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1445         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1446         req->state_sz = SHA256_DIGEST_SIZE;
1447         req->digest_sz = SHA256_DIGEST_SIZE;
1448         req->block_sz = SHA256_BLOCK_SIZE;
1449         req->hmac = true;
1450
1451         return 0;
1452 }
1453
1454 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1455 {
1456         int ret = safexcel_hmac_sha256_init(areq);
1457
1458         if (ret)
1459                 return ret;
1460
1461         return safexcel_ahash_finup(areq);
1462 }
1463
1464 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1465         .type = SAFEXCEL_ALG_TYPE_AHASH,
1466         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1467         .alg.ahash = {
1468                 .init = safexcel_hmac_sha256_init,
1469                 .update = safexcel_ahash_update,
1470                 .final = safexcel_ahash_final,
1471                 .finup = safexcel_ahash_finup,
1472                 .digest = safexcel_hmac_sha256_digest,
1473                 .setkey = safexcel_hmac_sha256_setkey,
1474                 .export = safexcel_ahash_export,
1475                 .import = safexcel_ahash_import,
1476                 .halg = {
1477                         .digestsize = SHA256_DIGEST_SIZE,
1478                         .statesize = sizeof(struct safexcel_ahash_export_state),
1479                         .base = {
1480                                 .cra_name = "hmac(sha256)",
1481                                 .cra_driver_name = "safexcel-hmac-sha256",
1482                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1483                                 .cra_flags = CRYPTO_ALG_ASYNC |
1484                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1485                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1486                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1487                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1488                                 .cra_init = safexcel_ahash_cra_init,
1489                                 .cra_exit = safexcel_ahash_cra_exit,
1490                                 .cra_module = THIS_MODULE,
1491                         },
1492                 },
1493         },
1494 };
1495
1496 static int safexcel_sha512_init(struct ahash_request *areq)
1497 {
1498         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1499         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1500
1501         memset(req, 0, sizeof(*req));
1502
1503         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1504         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1505         req->state_sz = SHA512_DIGEST_SIZE;
1506         req->digest_sz = SHA512_DIGEST_SIZE;
1507         req->block_sz = SHA512_BLOCK_SIZE;
1508
1509         return 0;
1510 }
1511
1512 static int safexcel_sha512_digest(struct ahash_request *areq)
1513 {
1514         int ret = safexcel_sha512_init(areq);
1515
1516         if (ret)
1517                 return ret;
1518
1519         return safexcel_ahash_finup(areq);
1520 }
1521
1522 struct safexcel_alg_template safexcel_alg_sha512 = {
1523         .type = SAFEXCEL_ALG_TYPE_AHASH,
1524         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1525         .alg.ahash = {
1526                 .init = safexcel_sha512_init,
1527                 .update = safexcel_ahash_update,
1528                 .final = safexcel_ahash_final,
1529                 .finup = safexcel_ahash_finup,
1530                 .digest = safexcel_sha512_digest,
1531                 .export = safexcel_ahash_export,
1532                 .import = safexcel_ahash_import,
1533                 .halg = {
1534                         .digestsize = SHA512_DIGEST_SIZE,
1535                         .statesize = sizeof(struct safexcel_ahash_export_state),
1536                         .base = {
1537                                 .cra_name = "sha512",
1538                                 .cra_driver_name = "safexcel-sha512",
1539                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1540                                 .cra_flags = CRYPTO_ALG_ASYNC |
1541                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1542                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1543                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1544                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1545                                 .cra_init = safexcel_ahash_cra_init,
1546                                 .cra_exit = safexcel_ahash_cra_exit,
1547                                 .cra_module = THIS_MODULE,
1548                         },
1549                 },
1550         },
1551 };
1552
1553 static int safexcel_sha384_init(struct ahash_request *areq)
1554 {
1555         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1556         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1557
1558         memset(req, 0, sizeof(*req));
1559
1560         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1561         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1562         req->state_sz = SHA512_DIGEST_SIZE;
1563         req->digest_sz = SHA512_DIGEST_SIZE;
1564         req->block_sz = SHA512_BLOCK_SIZE;
1565
1566         return 0;
1567 }
1568
1569 static int safexcel_sha384_digest(struct ahash_request *areq)
1570 {
1571         int ret = safexcel_sha384_init(areq);
1572
1573         if (ret)
1574                 return ret;
1575
1576         return safexcel_ahash_finup(areq);
1577 }
1578
1579 struct safexcel_alg_template safexcel_alg_sha384 = {
1580         .type = SAFEXCEL_ALG_TYPE_AHASH,
1581         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1582         .alg.ahash = {
1583                 .init = safexcel_sha384_init,
1584                 .update = safexcel_ahash_update,
1585                 .final = safexcel_ahash_final,
1586                 .finup = safexcel_ahash_finup,
1587                 .digest = safexcel_sha384_digest,
1588                 .export = safexcel_ahash_export,
1589                 .import = safexcel_ahash_import,
1590                 .halg = {
1591                         .digestsize = SHA384_DIGEST_SIZE,
1592                         .statesize = sizeof(struct safexcel_ahash_export_state),
1593                         .base = {
1594                                 .cra_name = "sha384",
1595                                 .cra_driver_name = "safexcel-sha384",
1596                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1597                                 .cra_flags = CRYPTO_ALG_ASYNC |
1598                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1599                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1600                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1601                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1602                                 .cra_init = safexcel_ahash_cra_init,
1603                                 .cra_exit = safexcel_ahash_cra_exit,
1604                                 .cra_module = THIS_MODULE,
1605                         },
1606                 },
1607         },
1608 };
1609
1610 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1611                                        unsigned int keylen)
1612 {
1613         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1614                                         SHA512_DIGEST_SIZE);
1615 }
1616
1617 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1618 {
1619         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1620         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1621
1622         memset(req, 0, sizeof(*req));
1623
1624         /* Start from ipad precompute */
1625         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1626         /* Already processed the key^ipad part now! */
1627         req->len        = SHA512_BLOCK_SIZE;
1628         req->processed  = SHA512_BLOCK_SIZE;
1629
1630         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1631         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1632         req->state_sz = SHA512_DIGEST_SIZE;
1633         req->digest_sz = SHA512_DIGEST_SIZE;
1634         req->block_sz = SHA512_BLOCK_SIZE;
1635         req->hmac = true;
1636
1637         return 0;
1638 }
1639
1640 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1641 {
1642         int ret = safexcel_hmac_sha512_init(areq);
1643
1644         if (ret)
1645                 return ret;
1646
1647         return safexcel_ahash_finup(areq);
1648 }
1649
1650 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1651         .type = SAFEXCEL_ALG_TYPE_AHASH,
1652         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1653         .alg.ahash = {
1654                 .init = safexcel_hmac_sha512_init,
1655                 .update = safexcel_ahash_update,
1656                 .final = safexcel_ahash_final,
1657                 .finup = safexcel_ahash_finup,
1658                 .digest = safexcel_hmac_sha512_digest,
1659                 .setkey = safexcel_hmac_sha512_setkey,
1660                 .export = safexcel_ahash_export,
1661                 .import = safexcel_ahash_import,
1662                 .halg = {
1663                         .digestsize = SHA512_DIGEST_SIZE,
1664                         .statesize = sizeof(struct safexcel_ahash_export_state),
1665                         .base = {
1666                                 .cra_name = "hmac(sha512)",
1667                                 .cra_driver_name = "safexcel-hmac-sha512",
1668                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1669                                 .cra_flags = CRYPTO_ALG_ASYNC |
1670                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1671                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1672                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1673                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1674                                 .cra_init = safexcel_ahash_cra_init,
1675                                 .cra_exit = safexcel_ahash_cra_exit,
1676                                 .cra_module = THIS_MODULE,
1677                         },
1678                 },
1679         },
1680 };
1681
1682 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1683                                        unsigned int keylen)
1684 {
1685         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1686                                         SHA512_DIGEST_SIZE);
1687 }
1688
1689 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1690 {
1691         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1692         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1693
1694         memset(req, 0, sizeof(*req));
1695
1696         /* Start from ipad precompute */
1697         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1698         /* Already processed the key^ipad part now! */
1699         req->len        = SHA512_BLOCK_SIZE;
1700         req->processed  = SHA512_BLOCK_SIZE;
1701
1702         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1703         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1704         req->state_sz = SHA512_DIGEST_SIZE;
1705         req->digest_sz = SHA512_DIGEST_SIZE;
1706         req->block_sz = SHA512_BLOCK_SIZE;
1707         req->hmac = true;
1708
1709         return 0;
1710 }
1711
1712 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1713 {
1714         int ret = safexcel_hmac_sha384_init(areq);
1715
1716         if (ret)
1717                 return ret;
1718
1719         return safexcel_ahash_finup(areq);
1720 }
1721
1722 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1723         .type = SAFEXCEL_ALG_TYPE_AHASH,
1724         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1725         .alg.ahash = {
1726                 .init = safexcel_hmac_sha384_init,
1727                 .update = safexcel_ahash_update,
1728                 .final = safexcel_ahash_final,
1729                 .finup = safexcel_ahash_finup,
1730                 .digest = safexcel_hmac_sha384_digest,
1731                 .setkey = safexcel_hmac_sha384_setkey,
1732                 .export = safexcel_ahash_export,
1733                 .import = safexcel_ahash_import,
1734                 .halg = {
1735                         .digestsize = SHA384_DIGEST_SIZE,
1736                         .statesize = sizeof(struct safexcel_ahash_export_state),
1737                         .base = {
1738                                 .cra_name = "hmac(sha384)",
1739                                 .cra_driver_name = "safexcel-hmac-sha384",
1740                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1741                                 .cra_flags = CRYPTO_ALG_ASYNC |
1742                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1743                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1744                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1745                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1746                                 .cra_init = safexcel_ahash_cra_init,
1747                                 .cra_exit = safexcel_ahash_cra_exit,
1748                                 .cra_module = THIS_MODULE,
1749                         },
1750                 },
1751         },
1752 };
1753
1754 static int safexcel_md5_init(struct ahash_request *areq)
1755 {
1756         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1757         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1758
1759         memset(req, 0, sizeof(*req));
1760
1761         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1762         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1763         req->state_sz = MD5_DIGEST_SIZE;
1764         req->digest_sz = MD5_DIGEST_SIZE;
1765         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1766
1767         return 0;
1768 }
1769
1770 static int safexcel_md5_digest(struct ahash_request *areq)
1771 {
1772         int ret = safexcel_md5_init(areq);
1773
1774         if (ret)
1775                 return ret;
1776
1777         return safexcel_ahash_finup(areq);
1778 }
1779
1780 struct safexcel_alg_template safexcel_alg_md5 = {
1781         .type = SAFEXCEL_ALG_TYPE_AHASH,
1782         .algo_mask = SAFEXCEL_ALG_MD5,
1783         .alg.ahash = {
1784                 .init = safexcel_md5_init,
1785                 .update = safexcel_ahash_update,
1786                 .final = safexcel_ahash_final,
1787                 .finup = safexcel_ahash_finup,
1788                 .digest = safexcel_md5_digest,
1789                 .export = safexcel_ahash_export,
1790                 .import = safexcel_ahash_import,
1791                 .halg = {
1792                         .digestsize = MD5_DIGEST_SIZE,
1793                         .statesize = sizeof(struct safexcel_ahash_export_state),
1794                         .base = {
1795                                 .cra_name = "md5",
1796                                 .cra_driver_name = "safexcel-md5",
1797                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1798                                 .cra_flags = CRYPTO_ALG_ASYNC |
1799                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1800                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1801                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1802                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1803                                 .cra_init = safexcel_ahash_cra_init,
1804                                 .cra_exit = safexcel_ahash_cra_exit,
1805                                 .cra_module = THIS_MODULE,
1806                         },
1807                 },
1808         },
1809 };
1810
1811 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1812 {
1813         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1814         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1815
1816         memset(req, 0, sizeof(*req));
1817
1818         /* Start from ipad precompute */
1819         memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1820         /* Already processed the key^ipad part now! */
1821         req->len        = MD5_HMAC_BLOCK_SIZE;
1822         req->processed  = MD5_HMAC_BLOCK_SIZE;
1823
1824         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1825         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1826         req->state_sz = MD5_DIGEST_SIZE;
1827         req->digest_sz = MD5_DIGEST_SIZE;
1828         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1829         req->len_is_le = true; /* MD5 is little endian! ... */
1830         req->hmac = true;
1831
1832         return 0;
1833 }
1834
1835 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1836                                      unsigned int keylen)
1837 {
1838         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1839                                         MD5_DIGEST_SIZE);
1840 }
1841
1842 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1843 {
1844         int ret = safexcel_hmac_md5_init(areq);
1845
1846         if (ret)
1847                 return ret;
1848
1849         return safexcel_ahash_finup(areq);
1850 }
1851
1852 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1853         .type = SAFEXCEL_ALG_TYPE_AHASH,
1854         .algo_mask = SAFEXCEL_ALG_MD5,
1855         .alg.ahash = {
1856                 .init = safexcel_hmac_md5_init,
1857                 .update = safexcel_ahash_update,
1858                 .final = safexcel_ahash_final,
1859                 .finup = safexcel_ahash_finup,
1860                 .digest = safexcel_hmac_md5_digest,
1861                 .setkey = safexcel_hmac_md5_setkey,
1862                 .export = safexcel_ahash_export,
1863                 .import = safexcel_ahash_import,
1864                 .halg = {
1865                         .digestsize = MD5_DIGEST_SIZE,
1866                         .statesize = sizeof(struct safexcel_ahash_export_state),
1867                         .base = {
1868                                 .cra_name = "hmac(md5)",
1869                                 .cra_driver_name = "safexcel-hmac-md5",
1870                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1871                                 .cra_flags = CRYPTO_ALG_ASYNC |
1872                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1873                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1874                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1875                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1876                                 .cra_init = safexcel_ahash_cra_init,
1877                                 .cra_exit = safexcel_ahash_cra_exit,
1878                                 .cra_module = THIS_MODULE,
1879                         },
1880                 },
1881         },
1882 };
1883
1884 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1885 {
1886         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1887         int ret = safexcel_ahash_cra_init(tfm);
1888
1889         /* Default 'key' is all zeroes */
1890         memset(&ctx->base.ipad, 0, sizeof(u32));
1891         return ret;
1892 }
1893
1894 static int safexcel_crc32_init(struct ahash_request *areq)
1895 {
1896         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1897         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1898
1899         memset(req, 0, sizeof(*req));
1900
1901         /* Start from loaded key */
1902         req->state[0]   = cpu_to_le32(~ctx->base.ipad.word[0]);
1903         /* Set processed to non-zero to enable invalidation detection */
1904         req->len        = sizeof(u32);
1905         req->processed  = sizeof(u32);
1906
1907         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1908         req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1909         req->state_sz = sizeof(u32);
1910         req->digest_sz = sizeof(u32);
1911         req->block_sz = sizeof(u32);
1912
1913         return 0;
1914 }
1915
1916 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1917                                  unsigned int keylen)
1918 {
1919         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1920
1921         if (keylen != sizeof(u32))
1922                 return -EINVAL;
1923
1924         memcpy(&ctx->base.ipad, key, sizeof(u32));
1925         return 0;
1926 }
1927
1928 static int safexcel_crc32_digest(struct ahash_request *areq)
1929 {
1930         return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1931 }
1932
1933 struct safexcel_alg_template safexcel_alg_crc32 = {
1934         .type = SAFEXCEL_ALG_TYPE_AHASH,
1935         .algo_mask = 0,
1936         .alg.ahash = {
1937                 .init = safexcel_crc32_init,
1938                 .update = safexcel_ahash_update,
1939                 .final = safexcel_ahash_final,
1940                 .finup = safexcel_ahash_finup,
1941                 .digest = safexcel_crc32_digest,
1942                 .setkey = safexcel_crc32_setkey,
1943                 .export = safexcel_ahash_export,
1944                 .import = safexcel_ahash_import,
1945                 .halg = {
1946                         .digestsize = sizeof(u32),
1947                         .statesize = sizeof(struct safexcel_ahash_export_state),
1948                         .base = {
1949                                 .cra_name = "crc32",
1950                                 .cra_driver_name = "safexcel-crc32",
1951                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1952                                 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1953                                              CRYPTO_ALG_ASYNC |
1954                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1955                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1956                                 .cra_blocksize = 1,
1957                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1958                                 .cra_init = safexcel_crc32_cra_init,
1959                                 .cra_exit = safexcel_ahash_cra_exit,
1960                                 .cra_module = THIS_MODULE,
1961                         },
1962                 },
1963         },
1964 };
1965
1966 static int safexcel_cbcmac_init(struct ahash_request *areq)
1967 {
1968         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1969         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1970
1971         memset(req, 0, sizeof(*req));
1972
1973         /* Start from loaded keys */
1974         memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
1975         /* Set processed to non-zero to enable invalidation detection */
1976         req->len        = AES_BLOCK_SIZE;
1977         req->processed  = AES_BLOCK_SIZE;
1978
1979         req->digest   = CONTEXT_CONTROL_DIGEST_XCM;
1980         req->state_sz = ctx->key_sz;
1981         req->digest_sz = AES_BLOCK_SIZE;
1982         req->block_sz = AES_BLOCK_SIZE;
1983         req->xcbcmac  = true;
1984
1985         return 0;
1986 }
1987
1988 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1989                                  unsigned int len)
1990 {
1991         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1992         struct crypto_aes_ctx aes;
1993         int ret, i;
1994
1995         ret = aes_expandkey(&aes, key, len);
1996         if (ret)
1997                 return ret;
1998
1999         memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2000         for (i = 0; i < len / sizeof(u32); i++)
2001                 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2002
2003         if (len == AES_KEYSIZE_192) {
2004                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2005                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2006         } else if (len == AES_KEYSIZE_256) {
2007                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2008                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2009         } else {
2010                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2011                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2012         }
2013         ctx->cbcmac  = true;
2014
2015         memzero_explicit(&aes, sizeof(aes));
2016         return 0;
2017 }
2018
2019 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2020 {
2021         return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2022 }
2023
2024 struct safexcel_alg_template safexcel_alg_cbcmac = {
2025         .type = SAFEXCEL_ALG_TYPE_AHASH,
2026         .algo_mask = 0,
2027         .alg.ahash = {
2028                 .init = safexcel_cbcmac_init,
2029                 .update = safexcel_ahash_update,
2030                 .final = safexcel_ahash_final,
2031                 .finup = safexcel_ahash_finup,
2032                 .digest = safexcel_cbcmac_digest,
2033                 .setkey = safexcel_cbcmac_setkey,
2034                 .export = safexcel_ahash_export,
2035                 .import = safexcel_ahash_import,
2036                 .halg = {
2037                         .digestsize = AES_BLOCK_SIZE,
2038                         .statesize = sizeof(struct safexcel_ahash_export_state),
2039                         .base = {
2040                                 .cra_name = "cbcmac(aes)",
2041                                 .cra_driver_name = "safexcel-cbcmac-aes",
2042                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2043                                 .cra_flags = CRYPTO_ALG_ASYNC |
2044                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2045                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2046                                 .cra_blocksize = 1,
2047                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2048                                 .cra_init = safexcel_ahash_cra_init,
2049                                 .cra_exit = safexcel_ahash_cra_exit,
2050                                 .cra_module = THIS_MODULE,
2051                         },
2052                 },
2053         },
2054 };
2055
2056 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2057                                  unsigned int len)
2058 {
2059         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2060         u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2061         int ret, i;
2062
2063         ret = aes_expandkey(ctx->aes, key, len);
2064         if (ret)
2065                 return ret;
2066
2067         /* precompute the XCBC key material */
2068         aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2069                     "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2070         aes_encrypt(ctx->aes, (u8 *)key_tmp,
2071                     "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2072         aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2073                     "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2074         for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2075                 ctx->base.ipad.word[i] = swab32(key_tmp[i]);
2076
2077         ret = aes_expandkey(ctx->aes,
2078                             (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2079                             AES_MIN_KEY_SIZE);
2080         if (ret)
2081                 return ret;
2082
2083         ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2084         ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2085         ctx->cbcmac = false;
2086
2087         return 0;
2088 }
2089
2090 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2091 {
2092         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2093
2094         safexcel_ahash_cra_init(tfm);
2095         ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL);
2096         return PTR_ERR_OR_ZERO(ctx->aes);
2097 }
2098
2099 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2100 {
2101         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2102
2103         kfree(ctx->aes);
2104         safexcel_ahash_cra_exit(tfm);
2105 }
2106
2107 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2108         .type = SAFEXCEL_ALG_TYPE_AHASH,
2109         .algo_mask = 0,
2110         .alg.ahash = {
2111                 .init = safexcel_cbcmac_init,
2112                 .update = safexcel_ahash_update,
2113                 .final = safexcel_ahash_final,
2114                 .finup = safexcel_ahash_finup,
2115                 .digest = safexcel_cbcmac_digest,
2116                 .setkey = safexcel_xcbcmac_setkey,
2117                 .export = safexcel_ahash_export,
2118                 .import = safexcel_ahash_import,
2119                 .halg = {
2120                         .digestsize = AES_BLOCK_SIZE,
2121                         .statesize = sizeof(struct safexcel_ahash_export_state),
2122                         .base = {
2123                                 .cra_name = "xcbc(aes)",
2124                                 .cra_driver_name = "safexcel-xcbc-aes",
2125                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2126                                 .cra_flags = CRYPTO_ALG_ASYNC |
2127                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2128                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2129                                 .cra_blocksize = AES_BLOCK_SIZE,
2130                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2131                                 .cra_init = safexcel_xcbcmac_cra_init,
2132                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2133                                 .cra_module = THIS_MODULE,
2134                         },
2135                 },
2136         },
2137 };
2138
2139 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2140                                 unsigned int len)
2141 {
2142         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2143         __be64 consts[4];
2144         u64 _const[2];
2145         u8 msb_mask, gfmask;
2146         int ret, i;
2147
2148         /* precompute the CMAC key material */
2149         ret = aes_expandkey(ctx->aes, key, len);
2150         if (ret)
2151                 return ret;
2152
2153         for (i = 0; i < len / sizeof(u32); i++)
2154                 ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]);
2155
2156         /* code below borrowed from crypto/cmac.c */
2157         /* encrypt the zero block */
2158         memset(consts, 0, AES_BLOCK_SIZE);
2159         aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts);
2160
2161         gfmask = 0x87;
2162         _const[0] = be64_to_cpu(consts[1]);
2163         _const[1] = be64_to_cpu(consts[0]);
2164
2165         /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2166         for (i = 0; i < 4; i += 2) {
2167                 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2168                 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2169                 _const[0] = (_const[0] << 1) ^ msb_mask;
2170
2171                 consts[i + 0] = cpu_to_be64(_const[1]);
2172                 consts[i + 1] = cpu_to_be64(_const[0]);
2173         }
2174         /* end of code borrowed from crypto/cmac.c */
2175
2176         for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2177                 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2178
2179         if (len == AES_KEYSIZE_192) {
2180                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2181                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2182         } else if (len == AES_KEYSIZE_256) {
2183                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2184                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2185         } else {
2186                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2187                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2188         }
2189         ctx->cbcmac = false;
2190
2191         return 0;
2192 }
2193
2194 struct safexcel_alg_template safexcel_alg_cmac = {
2195         .type = SAFEXCEL_ALG_TYPE_AHASH,
2196         .algo_mask = 0,
2197         .alg.ahash = {
2198                 .init = safexcel_cbcmac_init,
2199                 .update = safexcel_ahash_update,
2200                 .final = safexcel_ahash_final,
2201                 .finup = safexcel_ahash_finup,
2202                 .digest = safexcel_cbcmac_digest,
2203                 .setkey = safexcel_cmac_setkey,
2204                 .export = safexcel_ahash_export,
2205                 .import = safexcel_ahash_import,
2206                 .halg = {
2207                         .digestsize = AES_BLOCK_SIZE,
2208                         .statesize = sizeof(struct safexcel_ahash_export_state),
2209                         .base = {
2210                                 .cra_name = "cmac(aes)",
2211                                 .cra_driver_name = "safexcel-cmac-aes",
2212                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2213                                 .cra_flags = CRYPTO_ALG_ASYNC |
2214                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2215                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2216                                 .cra_blocksize = AES_BLOCK_SIZE,
2217                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2218                                 .cra_init = safexcel_xcbcmac_cra_init,
2219                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2220                                 .cra_module = THIS_MODULE,
2221                         },
2222                 },
2223         },
2224 };
2225
2226 static int safexcel_sm3_init(struct ahash_request *areq)
2227 {
2228         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2229         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2230
2231         memset(req, 0, sizeof(*req));
2232
2233         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2234         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2235         req->state_sz = SM3_DIGEST_SIZE;
2236         req->digest_sz = SM3_DIGEST_SIZE;
2237         req->block_sz = SM3_BLOCK_SIZE;
2238
2239         return 0;
2240 }
2241
2242 static int safexcel_sm3_digest(struct ahash_request *areq)
2243 {
2244         int ret = safexcel_sm3_init(areq);
2245
2246         if (ret)
2247                 return ret;
2248
2249         return safexcel_ahash_finup(areq);
2250 }
2251
2252 struct safexcel_alg_template safexcel_alg_sm3 = {
2253         .type = SAFEXCEL_ALG_TYPE_AHASH,
2254         .algo_mask = SAFEXCEL_ALG_SM3,
2255         .alg.ahash = {
2256                 .init = safexcel_sm3_init,
2257                 .update = safexcel_ahash_update,
2258                 .final = safexcel_ahash_final,
2259                 .finup = safexcel_ahash_finup,
2260                 .digest = safexcel_sm3_digest,
2261                 .export = safexcel_ahash_export,
2262                 .import = safexcel_ahash_import,
2263                 .halg = {
2264                         .digestsize = SM3_DIGEST_SIZE,
2265                         .statesize = sizeof(struct safexcel_ahash_export_state),
2266                         .base = {
2267                                 .cra_name = "sm3",
2268                                 .cra_driver_name = "safexcel-sm3",
2269                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2270                                 .cra_flags = CRYPTO_ALG_ASYNC |
2271                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2272                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2273                                 .cra_blocksize = SM3_BLOCK_SIZE,
2274                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2275                                 .cra_init = safexcel_ahash_cra_init,
2276                                 .cra_exit = safexcel_ahash_cra_exit,
2277                                 .cra_module = THIS_MODULE,
2278                         },
2279                 },
2280         },
2281 };
2282
2283 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2284                                     unsigned int keylen)
2285 {
2286         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2287                                         SM3_DIGEST_SIZE);
2288 }
2289
2290 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2291 {
2292         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2293         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2294
2295         memset(req, 0, sizeof(*req));
2296
2297         /* Start from ipad precompute */
2298         memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2299         /* Already processed the key^ipad part now! */
2300         req->len        = SM3_BLOCK_SIZE;
2301         req->processed  = SM3_BLOCK_SIZE;
2302
2303         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2304         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2305         req->state_sz = SM3_DIGEST_SIZE;
2306         req->digest_sz = SM3_DIGEST_SIZE;
2307         req->block_sz = SM3_BLOCK_SIZE;
2308         req->hmac = true;
2309
2310         return 0;
2311 }
2312
2313 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2314 {
2315         int ret = safexcel_hmac_sm3_init(areq);
2316
2317         if (ret)
2318                 return ret;
2319
2320         return safexcel_ahash_finup(areq);
2321 }
2322
2323 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2324         .type = SAFEXCEL_ALG_TYPE_AHASH,
2325         .algo_mask = SAFEXCEL_ALG_SM3,
2326         .alg.ahash = {
2327                 .init = safexcel_hmac_sm3_init,
2328                 .update = safexcel_ahash_update,
2329                 .final = safexcel_ahash_final,
2330                 .finup = safexcel_ahash_finup,
2331                 .digest = safexcel_hmac_sm3_digest,
2332                 .setkey = safexcel_hmac_sm3_setkey,
2333                 .export = safexcel_ahash_export,
2334                 .import = safexcel_ahash_import,
2335                 .halg = {
2336                         .digestsize = SM3_DIGEST_SIZE,
2337                         .statesize = sizeof(struct safexcel_ahash_export_state),
2338                         .base = {
2339                                 .cra_name = "hmac(sm3)",
2340                                 .cra_driver_name = "safexcel-hmac-sm3",
2341                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2342                                 .cra_flags = CRYPTO_ALG_ASYNC |
2343                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2344                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2345                                 .cra_blocksize = SM3_BLOCK_SIZE,
2346                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2347                                 .cra_init = safexcel_ahash_cra_init,
2348                                 .cra_exit = safexcel_ahash_cra_exit,
2349                                 .cra_module = THIS_MODULE,
2350                         },
2351                 },
2352         },
2353 };
2354
2355 static int safexcel_sha3_224_init(struct ahash_request *areq)
2356 {
2357         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2358         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2359         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2360
2361         memset(req, 0, sizeof(*req));
2362
2363         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2364         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2365         req->state_sz = SHA3_224_DIGEST_SIZE;
2366         req->digest_sz = SHA3_224_DIGEST_SIZE;
2367         req->block_sz = SHA3_224_BLOCK_SIZE;
2368         ctx->do_fallback = false;
2369         ctx->fb_init_done = false;
2370         return 0;
2371 }
2372
2373 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2374 {
2375         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2376         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2377         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2378         int ret = 0;
2379
2380         if (ctx->do_fallback) {
2381                 ahash_request_set_tfm(subreq, ctx->fback);
2382                 ahash_request_set_callback(subreq, req->base.flags,
2383                                            req->base.complete, req->base.data);
2384                 ahash_request_set_crypt(subreq, req->src, req->result,
2385                                         req->nbytes);
2386                 if (!ctx->fb_init_done) {
2387                         if (ctx->fb_do_setkey) {
2388                                 /* Set fallback cipher HMAC key */
2389                                 u8 key[SHA3_224_BLOCK_SIZE];
2390
2391                                 memcpy(key, &ctx->base.ipad,
2392                                        crypto_ahash_blocksize(ctx->fback) / 2);
2393                                 memcpy(key +
2394                                        crypto_ahash_blocksize(ctx->fback) / 2,
2395                                        &ctx->base.opad,
2396                                        crypto_ahash_blocksize(ctx->fback) / 2);
2397                                 ret = crypto_ahash_setkey(ctx->fback, key,
2398                                         crypto_ahash_blocksize(ctx->fback));
2399                                 memzero_explicit(key,
2400                                         crypto_ahash_blocksize(ctx->fback));
2401                                 ctx->fb_do_setkey = false;
2402                         }
2403                         ret = ret ?: crypto_ahash_init(subreq);
2404                         ctx->fb_init_done = true;
2405                 }
2406         }
2407         return ret;
2408 }
2409
2410 static int safexcel_sha3_update(struct ahash_request *req)
2411 {
2412         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2413         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2414         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2415
2416         ctx->do_fallback = true;
2417         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2418 }
2419
2420 static int safexcel_sha3_final(struct ahash_request *req)
2421 {
2422         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2423         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2424         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2425
2426         ctx->do_fallback = true;
2427         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2428 }
2429
2430 static int safexcel_sha3_finup(struct ahash_request *req)
2431 {
2432         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2433         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2434         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2435
2436         ctx->do_fallback |= !req->nbytes;
2437         if (ctx->do_fallback)
2438                 /* Update or ex/import happened or len 0, cannot use the HW */
2439                 return safexcel_sha3_fbcheck(req) ?:
2440                        crypto_ahash_finup(subreq);
2441         else
2442                 return safexcel_ahash_finup(req);
2443 }
2444
2445 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2446 {
2447         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2448         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2449         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2450
2451         ctx->do_fallback = true;
2452         ctx->fb_init_done = false;
2453         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2454 }
2455
2456 static int safexcel_sha3_224_digest(struct ahash_request *req)
2457 {
2458         if (req->nbytes)
2459                 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2460
2461         /* HW cannot do zero length hash, use fallback instead */
2462         return safexcel_sha3_digest_fallback(req);
2463 }
2464
2465 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2466 {
2467         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2468         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2469         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2470
2471         ctx->do_fallback = true;
2472         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2473 }
2474
2475 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2476 {
2477         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2478         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2479         struct ahash_request *subreq = ahash_request_ctx_dma(req);
2480
2481         ctx->do_fallback = true;
2482         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2483         // return safexcel_ahash_import(req, in);
2484 }
2485
2486 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2487 {
2488         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2489         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2490
2491         safexcel_ahash_cra_init(tfm);
2492
2493         /* Allocate fallback implementation */
2494         ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2495                                         CRYPTO_ALG_ASYNC |
2496                                         CRYPTO_ALG_NEED_FALLBACK);
2497         if (IS_ERR(ctx->fback))
2498                 return PTR_ERR(ctx->fback);
2499
2500         /* Update statesize from fallback algorithm! */
2501         crypto_hash_alg_common(ahash)->statesize =
2502                 crypto_ahash_statesize(ctx->fback);
2503         crypto_ahash_set_reqsize_dma(
2504                 ahash, max(sizeof(struct safexcel_ahash_req),
2505                            sizeof(struct ahash_request) +
2506                            crypto_ahash_reqsize(ctx->fback)));
2507         return 0;
2508 }
2509
2510 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2511 {
2512         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2513
2514         crypto_free_ahash(ctx->fback);
2515         safexcel_ahash_cra_exit(tfm);
2516 }
2517
2518 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2519         .type = SAFEXCEL_ALG_TYPE_AHASH,
2520         .algo_mask = SAFEXCEL_ALG_SHA3,
2521         .alg.ahash = {
2522                 .init = safexcel_sha3_224_init,
2523                 .update = safexcel_sha3_update,
2524                 .final = safexcel_sha3_final,
2525                 .finup = safexcel_sha3_finup,
2526                 .digest = safexcel_sha3_224_digest,
2527                 .export = safexcel_sha3_export,
2528                 .import = safexcel_sha3_import,
2529                 .halg = {
2530                         .digestsize = SHA3_224_DIGEST_SIZE,
2531                         .statesize = sizeof(struct safexcel_ahash_export_state),
2532                         .base = {
2533                                 .cra_name = "sha3-224",
2534                                 .cra_driver_name = "safexcel-sha3-224",
2535                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2536                                 .cra_flags = CRYPTO_ALG_ASYNC |
2537                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2538                                              CRYPTO_ALG_NEED_FALLBACK,
2539                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2540                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2541                                 .cra_init = safexcel_sha3_cra_init,
2542                                 .cra_exit = safexcel_sha3_cra_exit,
2543                                 .cra_module = THIS_MODULE,
2544                         },
2545                 },
2546         },
2547 };
2548
2549 static int safexcel_sha3_256_init(struct ahash_request *areq)
2550 {
2551         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2552         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2553         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2554
2555         memset(req, 0, sizeof(*req));
2556
2557         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2558         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2559         req->state_sz = SHA3_256_DIGEST_SIZE;
2560         req->digest_sz = SHA3_256_DIGEST_SIZE;
2561         req->block_sz = SHA3_256_BLOCK_SIZE;
2562         ctx->do_fallback = false;
2563         ctx->fb_init_done = false;
2564         return 0;
2565 }
2566
2567 static int safexcel_sha3_256_digest(struct ahash_request *req)
2568 {
2569         if (req->nbytes)
2570                 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2571
2572         /* HW cannot do zero length hash, use fallback instead */
2573         return safexcel_sha3_digest_fallback(req);
2574 }
2575
2576 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2577         .type = SAFEXCEL_ALG_TYPE_AHASH,
2578         .algo_mask = SAFEXCEL_ALG_SHA3,
2579         .alg.ahash = {
2580                 .init = safexcel_sha3_256_init,
2581                 .update = safexcel_sha3_update,
2582                 .final = safexcel_sha3_final,
2583                 .finup = safexcel_sha3_finup,
2584                 .digest = safexcel_sha3_256_digest,
2585                 .export = safexcel_sha3_export,
2586                 .import = safexcel_sha3_import,
2587                 .halg = {
2588                         .digestsize = SHA3_256_DIGEST_SIZE,
2589                         .statesize = sizeof(struct safexcel_ahash_export_state),
2590                         .base = {
2591                                 .cra_name = "sha3-256",
2592                                 .cra_driver_name = "safexcel-sha3-256",
2593                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2594                                 .cra_flags = CRYPTO_ALG_ASYNC |
2595                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2596                                              CRYPTO_ALG_NEED_FALLBACK,
2597                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2598                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2599                                 .cra_init = safexcel_sha3_cra_init,
2600                                 .cra_exit = safexcel_sha3_cra_exit,
2601                                 .cra_module = THIS_MODULE,
2602                         },
2603                 },
2604         },
2605 };
2606
2607 static int safexcel_sha3_384_init(struct ahash_request *areq)
2608 {
2609         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2610         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2611         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2612
2613         memset(req, 0, sizeof(*req));
2614
2615         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2616         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2617         req->state_sz = SHA3_384_DIGEST_SIZE;
2618         req->digest_sz = SHA3_384_DIGEST_SIZE;
2619         req->block_sz = SHA3_384_BLOCK_SIZE;
2620         ctx->do_fallback = false;
2621         ctx->fb_init_done = false;
2622         return 0;
2623 }
2624
2625 static int safexcel_sha3_384_digest(struct ahash_request *req)
2626 {
2627         if (req->nbytes)
2628                 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2629
2630         /* HW cannot do zero length hash, use fallback instead */
2631         return safexcel_sha3_digest_fallback(req);
2632 }
2633
2634 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2635         .type = SAFEXCEL_ALG_TYPE_AHASH,
2636         .algo_mask = SAFEXCEL_ALG_SHA3,
2637         .alg.ahash = {
2638                 .init = safexcel_sha3_384_init,
2639                 .update = safexcel_sha3_update,
2640                 .final = safexcel_sha3_final,
2641                 .finup = safexcel_sha3_finup,
2642                 .digest = safexcel_sha3_384_digest,
2643                 .export = safexcel_sha3_export,
2644                 .import = safexcel_sha3_import,
2645                 .halg = {
2646                         .digestsize = SHA3_384_DIGEST_SIZE,
2647                         .statesize = sizeof(struct safexcel_ahash_export_state),
2648                         .base = {
2649                                 .cra_name = "sha3-384",
2650                                 .cra_driver_name = "safexcel-sha3-384",
2651                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2652                                 .cra_flags = CRYPTO_ALG_ASYNC |
2653                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2654                                              CRYPTO_ALG_NEED_FALLBACK,
2655                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2656                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2657                                 .cra_init = safexcel_sha3_cra_init,
2658                                 .cra_exit = safexcel_sha3_cra_exit,
2659                                 .cra_module = THIS_MODULE,
2660                         },
2661                 },
2662         },
2663 };
2664
2665 static int safexcel_sha3_512_init(struct ahash_request *areq)
2666 {
2667         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2668         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2669         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2670
2671         memset(req, 0, sizeof(*req));
2672
2673         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2674         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2675         req->state_sz = SHA3_512_DIGEST_SIZE;
2676         req->digest_sz = SHA3_512_DIGEST_SIZE;
2677         req->block_sz = SHA3_512_BLOCK_SIZE;
2678         ctx->do_fallback = false;
2679         ctx->fb_init_done = false;
2680         return 0;
2681 }
2682
2683 static int safexcel_sha3_512_digest(struct ahash_request *req)
2684 {
2685         if (req->nbytes)
2686                 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2687
2688         /* HW cannot do zero length hash, use fallback instead */
2689         return safexcel_sha3_digest_fallback(req);
2690 }
2691
2692 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2693         .type = SAFEXCEL_ALG_TYPE_AHASH,
2694         .algo_mask = SAFEXCEL_ALG_SHA3,
2695         .alg.ahash = {
2696                 .init = safexcel_sha3_512_init,
2697                 .update = safexcel_sha3_update,
2698                 .final = safexcel_sha3_final,
2699                 .finup = safexcel_sha3_finup,
2700                 .digest = safexcel_sha3_512_digest,
2701                 .export = safexcel_sha3_export,
2702                 .import = safexcel_sha3_import,
2703                 .halg = {
2704                         .digestsize = SHA3_512_DIGEST_SIZE,
2705                         .statesize = sizeof(struct safexcel_ahash_export_state),
2706                         .base = {
2707                                 .cra_name = "sha3-512",
2708                                 .cra_driver_name = "safexcel-sha3-512",
2709                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2710                                 .cra_flags = CRYPTO_ALG_ASYNC |
2711                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2712                                              CRYPTO_ALG_NEED_FALLBACK,
2713                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2714                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2715                                 .cra_init = safexcel_sha3_cra_init,
2716                                 .cra_exit = safexcel_sha3_cra_exit,
2717                                 .cra_module = THIS_MODULE,
2718                         },
2719                 },
2720         },
2721 };
2722
2723 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2724 {
2725         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2726         int ret;
2727
2728         ret = safexcel_sha3_cra_init(tfm);
2729         if (ret)
2730                 return ret;
2731
2732         /* Allocate precalc basic digest implementation */
2733         ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2734         if (IS_ERR(ctx->shpre))
2735                 return PTR_ERR(ctx->shpre);
2736
2737         ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2738                               crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2739         if (!ctx->shdesc) {
2740                 crypto_free_shash(ctx->shpre);
2741                 return -ENOMEM;
2742         }
2743         ctx->shdesc->tfm = ctx->shpre;
2744         return 0;
2745 }
2746
2747 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2748 {
2749         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2750
2751         crypto_free_ahash(ctx->fback);
2752         crypto_free_shash(ctx->shpre);
2753         kfree(ctx->shdesc);
2754         safexcel_ahash_cra_exit(tfm);
2755 }
2756
2757 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2758                                      unsigned int keylen)
2759 {
2760         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2761         int ret = 0;
2762
2763         if (keylen > crypto_ahash_blocksize(tfm)) {
2764                 /*
2765                  * If the key is larger than the blocksize, then hash it
2766                  * first using our fallback cipher
2767                  */
2768                 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2769                                           ctx->base.ipad.byte);
2770                 keylen = crypto_shash_digestsize(ctx->shpre);
2771
2772                 /*
2773                  * If the digest is larger than half the blocksize, we need to
2774                  * move the rest to opad due to the way our HMAC infra works.
2775                  */
2776                 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2777                         /* Buffers overlap, need to use memmove iso memcpy! */
2778                         memmove(&ctx->base.opad,
2779                                 ctx->base.ipad.byte +
2780                                         crypto_ahash_blocksize(tfm) / 2,
2781                                 keylen - crypto_ahash_blocksize(tfm) / 2);
2782         } else {
2783                 /*
2784                  * Copy the key to our ipad & opad buffers
2785                  * Note that ipad and opad each contain one half of the key,
2786                  * to match the existing HMAC driver infrastructure.
2787                  */
2788                 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2789                         memcpy(&ctx->base.ipad, key, keylen);
2790                 } else {
2791                         memcpy(&ctx->base.ipad, key,
2792                                crypto_ahash_blocksize(tfm) / 2);
2793                         memcpy(&ctx->base.opad,
2794                                key + crypto_ahash_blocksize(tfm) / 2,
2795                                keylen - crypto_ahash_blocksize(tfm) / 2);
2796                 }
2797         }
2798
2799         /* Pad key with zeroes */
2800         if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2801                 memset(ctx->base.ipad.byte + keylen, 0,
2802                        crypto_ahash_blocksize(tfm) / 2 - keylen);
2803                 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2804         } else {
2805                 memset(ctx->base.opad.byte + keylen -
2806                        crypto_ahash_blocksize(tfm) / 2, 0,
2807                        crypto_ahash_blocksize(tfm) - keylen);
2808         }
2809
2810         /* If doing fallback, still need to set the new key! */
2811         ctx->fb_do_setkey = true;
2812         return ret;
2813 }
2814
2815 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2816 {
2817         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2818         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2819         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2820
2821         memset(req, 0, sizeof(*req));
2822
2823         /* Copy (half of) the key */
2824         memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2825         /* Start of HMAC should have len == processed == blocksize */
2826         req->len        = SHA3_224_BLOCK_SIZE;
2827         req->processed  = SHA3_224_BLOCK_SIZE;
2828         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2829         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2830         req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2831         req->digest_sz = SHA3_224_DIGEST_SIZE;
2832         req->block_sz = SHA3_224_BLOCK_SIZE;
2833         req->hmac = true;
2834         ctx->do_fallback = false;
2835         ctx->fb_init_done = false;
2836         return 0;
2837 }
2838
2839 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2840 {
2841         if (req->nbytes)
2842                 return safexcel_hmac_sha3_224_init(req) ?:
2843                        safexcel_ahash_finup(req);
2844
2845         /* HW cannot do zero length HMAC, use fallback instead */
2846         return safexcel_sha3_digest_fallback(req);
2847 }
2848
2849 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2850 {
2851         return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2852 }
2853
2854 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2855         .type = SAFEXCEL_ALG_TYPE_AHASH,
2856         .algo_mask = SAFEXCEL_ALG_SHA3,
2857         .alg.ahash = {
2858                 .init = safexcel_hmac_sha3_224_init,
2859                 .update = safexcel_sha3_update,
2860                 .final = safexcel_sha3_final,
2861                 .finup = safexcel_sha3_finup,
2862                 .digest = safexcel_hmac_sha3_224_digest,
2863                 .setkey = safexcel_hmac_sha3_setkey,
2864                 .export = safexcel_sha3_export,
2865                 .import = safexcel_sha3_import,
2866                 .halg = {
2867                         .digestsize = SHA3_224_DIGEST_SIZE,
2868                         .statesize = sizeof(struct safexcel_ahash_export_state),
2869                         .base = {
2870                                 .cra_name = "hmac(sha3-224)",
2871                                 .cra_driver_name = "safexcel-hmac-sha3-224",
2872                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2873                                 .cra_flags = CRYPTO_ALG_ASYNC |
2874                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2875                                              CRYPTO_ALG_NEED_FALLBACK,
2876                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2877                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2878                                 .cra_init = safexcel_hmac_sha3_224_cra_init,
2879                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2880                                 .cra_module = THIS_MODULE,
2881                         },
2882                 },
2883         },
2884 };
2885
2886 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2887 {
2888         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2889         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2890         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2891
2892         memset(req, 0, sizeof(*req));
2893
2894         /* Copy (half of) the key */
2895         memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2896         /* Start of HMAC should have len == processed == blocksize */
2897         req->len        = SHA3_256_BLOCK_SIZE;
2898         req->processed  = SHA3_256_BLOCK_SIZE;
2899         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2900         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2901         req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2902         req->digest_sz = SHA3_256_DIGEST_SIZE;
2903         req->block_sz = SHA3_256_BLOCK_SIZE;
2904         req->hmac = true;
2905         ctx->do_fallback = false;
2906         ctx->fb_init_done = false;
2907         return 0;
2908 }
2909
2910 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2911 {
2912         if (req->nbytes)
2913                 return safexcel_hmac_sha3_256_init(req) ?:
2914                        safexcel_ahash_finup(req);
2915
2916         /* HW cannot do zero length HMAC, use fallback instead */
2917         return safexcel_sha3_digest_fallback(req);
2918 }
2919
2920 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2921 {
2922         return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2923 }
2924
2925 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2926         .type = SAFEXCEL_ALG_TYPE_AHASH,
2927         .algo_mask = SAFEXCEL_ALG_SHA3,
2928         .alg.ahash = {
2929                 .init = safexcel_hmac_sha3_256_init,
2930                 .update = safexcel_sha3_update,
2931                 .final = safexcel_sha3_final,
2932                 .finup = safexcel_sha3_finup,
2933                 .digest = safexcel_hmac_sha3_256_digest,
2934                 .setkey = safexcel_hmac_sha3_setkey,
2935                 .export = safexcel_sha3_export,
2936                 .import = safexcel_sha3_import,
2937                 .halg = {
2938                         .digestsize = SHA3_256_DIGEST_SIZE,
2939                         .statesize = sizeof(struct safexcel_ahash_export_state),
2940                         .base = {
2941                                 .cra_name = "hmac(sha3-256)",
2942                                 .cra_driver_name = "safexcel-hmac-sha3-256",
2943                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2944                                 .cra_flags = CRYPTO_ALG_ASYNC |
2945                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2946                                              CRYPTO_ALG_NEED_FALLBACK,
2947                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2948                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2949                                 .cra_init = safexcel_hmac_sha3_256_cra_init,
2950                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2951                                 .cra_module = THIS_MODULE,
2952                         },
2953                 },
2954         },
2955 };
2956
2957 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
2958 {
2959         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2960         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2961         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2962
2963         memset(req, 0, sizeof(*req));
2964
2965         /* Copy (half of) the key */
2966         memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
2967         /* Start of HMAC should have len == processed == blocksize */
2968         req->len        = SHA3_384_BLOCK_SIZE;
2969         req->processed  = SHA3_384_BLOCK_SIZE;
2970         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2971         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2972         req->state_sz = SHA3_384_BLOCK_SIZE / 2;
2973         req->digest_sz = SHA3_384_DIGEST_SIZE;
2974         req->block_sz = SHA3_384_BLOCK_SIZE;
2975         req->hmac = true;
2976         ctx->do_fallback = false;
2977         ctx->fb_init_done = false;
2978         return 0;
2979 }
2980
2981 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
2982 {
2983         if (req->nbytes)
2984                 return safexcel_hmac_sha3_384_init(req) ?:
2985                        safexcel_ahash_finup(req);
2986
2987         /* HW cannot do zero length HMAC, use fallback instead */
2988         return safexcel_sha3_digest_fallback(req);
2989 }
2990
2991 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
2992 {
2993         return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
2994 }
2995
2996 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
2997         .type = SAFEXCEL_ALG_TYPE_AHASH,
2998         .algo_mask = SAFEXCEL_ALG_SHA3,
2999         .alg.ahash = {
3000                 .init = safexcel_hmac_sha3_384_init,
3001                 .update = safexcel_sha3_update,
3002                 .final = safexcel_sha3_final,
3003                 .finup = safexcel_sha3_finup,
3004                 .digest = safexcel_hmac_sha3_384_digest,
3005                 .setkey = safexcel_hmac_sha3_setkey,
3006                 .export = safexcel_sha3_export,
3007                 .import = safexcel_sha3_import,
3008                 .halg = {
3009                         .digestsize = SHA3_384_DIGEST_SIZE,
3010                         .statesize = sizeof(struct safexcel_ahash_export_state),
3011                         .base = {
3012                                 .cra_name = "hmac(sha3-384)",
3013                                 .cra_driver_name = "safexcel-hmac-sha3-384",
3014                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3015                                 .cra_flags = CRYPTO_ALG_ASYNC |
3016                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3017                                              CRYPTO_ALG_NEED_FALLBACK,
3018                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3019                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3020                                 .cra_init = safexcel_hmac_sha3_384_cra_init,
3021                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3022                                 .cra_module = THIS_MODULE,
3023                         },
3024                 },
3025         },
3026 };
3027
3028 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3029 {
3030         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3031         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3032         struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
3033
3034         memset(req, 0, sizeof(*req));
3035
3036         /* Copy (half of) the key */
3037         memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3038         /* Start of HMAC should have len == processed == blocksize */
3039         req->len        = SHA3_512_BLOCK_SIZE;
3040         req->processed  = SHA3_512_BLOCK_SIZE;
3041         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3042         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3043         req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3044         req->digest_sz = SHA3_512_DIGEST_SIZE;
3045         req->block_sz = SHA3_512_BLOCK_SIZE;
3046         req->hmac = true;
3047         ctx->do_fallback = false;
3048         ctx->fb_init_done = false;
3049         return 0;
3050 }
3051
3052 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3053 {
3054         if (req->nbytes)
3055                 return safexcel_hmac_sha3_512_init(req) ?:
3056                        safexcel_ahash_finup(req);
3057
3058         /* HW cannot do zero length HMAC, use fallback instead */
3059         return safexcel_sha3_digest_fallback(req);
3060 }
3061
3062 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3063 {
3064         return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3065 }
3066 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3067         .type = SAFEXCEL_ALG_TYPE_AHASH,
3068         .algo_mask = SAFEXCEL_ALG_SHA3,
3069         .alg.ahash = {
3070                 .init = safexcel_hmac_sha3_512_init,
3071                 .update = safexcel_sha3_update,
3072                 .final = safexcel_sha3_final,
3073                 .finup = safexcel_sha3_finup,
3074                 .digest = safexcel_hmac_sha3_512_digest,
3075                 .setkey = safexcel_hmac_sha3_setkey,
3076                 .export = safexcel_sha3_export,
3077                 .import = safexcel_sha3_import,
3078                 .halg = {
3079                         .digestsize = SHA3_512_DIGEST_SIZE,
3080                         .statesize = sizeof(struct safexcel_ahash_export_state),
3081                         .base = {
3082                                 .cra_name = "hmac(sha3-512)",
3083                                 .cra_driver_name = "safexcel-hmac-sha3-512",
3084                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3085                                 .cra_flags = CRYPTO_ALG_ASYNC |
3086                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3087                                              CRYPTO_ALG_NEED_FALLBACK,
3088                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3089                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3090                                 .cra_init = safexcel_hmac_sha3_512_cra_init,
3091                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3092                                 .cra_module = THIS_MODULE,
3093                         },
3094                 },
3095         },
3096 };