drivers/crypto/inside-secure/safexcel_hash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <antoine.tenart@free-electrons.com>
   6  */
   7
   8 #include <crypto/aes.h>
   9 #include <crypto/hmac.h>
  10 #include <crypto/md5.h>
  11 #include <crypto/sha1.h>
  12 #include <crypto/sha2.h>
  13 #include <crypto/sha3.h>
  14 #include <crypto/skcipher.h>
  15 #include <crypto/sm3.h>
  16 #include <crypto/internal/cipher.h>
  17 #include <linux/device.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/dmapool.h>
  20
  21 #include "safexcel.h"
  22
  23 struct safexcel_ahash_ctx {
  24         struct safexcel_context base;
  25
  26         u32 alg;
  27         u8  key_sz;
  28         bool cbcmac;
  29         bool do_fallback;
  30         bool fb_init_done;
  31         bool fb_do_setkey;
  32
  33         struct crypto_cipher *kaes;
  34         struct crypto_ahash *fback;
  35         struct crypto_shash *shpre;
  36         struct shash_desc *shdesc;
  37 };
  38
  39 struct safexcel_ahash_req {
  40         bool last_req;
  41         bool finish;
  42         bool hmac;
  43         bool needs_inv;
  44         bool hmac_zlen;
  45         bool len_is_le;
  46         bool not_first;
  47         bool xcbcmac;
  48
  49         int nents;
  50         dma_addr_t result_dma;
  51
  52         u32 digest;
  53
  54         u8 state_sz;    /* expected state size, only set once */
  55         u8 block_sz;    /* block size, only set once */
  56         u8 digest_sz;   /* output digest size, only set once */
  57         __le32 state[SHA3_512_BLOCK_SIZE /
  58                      sizeof(__le32)] __aligned(sizeof(__le32));
  59
  60         u64 len;
  61         u64 processed;
  62
  63         u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  64         dma_addr_t cache_dma;
  65         unsigned int cache_sz;
  66
  67         u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  68 };
  69
  70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
  71 {
  72         return req->len - req->processed;
  73 }
  74
  75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
  76                                 u32 input_length, u32 result_length,
  77                                 bool cbcmac)
  78 {
  79         struct safexcel_token *token =
  80                 (struct safexcel_token *)cdesc->control_data.token;
  81
  82         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  83         token[0].packet_length = input_length;
  84         token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  85
  86         input_length &= 15;
  87         if (unlikely(cbcmac && input_length)) {
  88                 token[0].stat =  0;
  89                 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
  90                 token[1].packet_length = 16 - input_length;
  91                 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
  92                 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  93         } else {
  94                 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
  95                 eip197_noop_token(&token[1]);
  96         }
  97
  98         token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
  99         token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
 100                         EIP197_TOKEN_STAT_LAST_PACKET;
 101         token[2].packet_length = result_length;
 102         token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 103                                 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 104
 105         eip197_noop_token(&token[3]);
 106 }
 107
 108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
 109                                      struct safexcel_ahash_req *req,
 110                                      struct safexcel_command_desc *cdesc)
 111 {
 112         struct safexcel_crypto_priv *priv = ctx->base.priv;
 113         u64 count = 0;
 114
 115         cdesc->control_data.control0 = ctx->alg;
 116         cdesc->control_data.control1 = 0;
 117
 118         /*
 119          * Copy the input digest if needed, and setup the context
 120          * fields. Do this now as we need it to setup the first command
 121          * descriptor.
 122          */
 123         if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
 124                 if (req->xcbcmac)
 125                         memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
 126                 else
 127                         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 128
 129                 if (!req->finish && req->xcbcmac)
 130                         cdesc->control_data.control0 |=
 131                                 CONTEXT_CONTROL_DIGEST_XCM |
 132                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 133                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 134                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 135                                                      sizeof(u32));
 136                 else
 137                         cdesc->control_data.control0 |=
 138                                 CONTEXT_CONTROL_DIGEST_XCM |
 139                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 140                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 141                                                      sizeof(u32));
 142                 return;
 143         } else if (!req->processed) {
 144                 /* First - and possibly only - block of basic hash only */
 145                 if (req->finish)
 146                         cdesc->control_data.control0 |= req->digest |
 147                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 148                                 CONTEXT_CONTROL_RESTART_HASH  |
 149                                 /* ensure its not 0! */
 150                                 CONTEXT_CONTROL_SIZE(1);
 151                 else
 152                         cdesc->control_data.control0 |= req->digest |
 153                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 154                                 CONTEXT_CONTROL_RESTART_HASH   |
 155                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 156                                 /* ensure its not 0! */
 157                                 CONTEXT_CONTROL_SIZE(1);
 158                 return;
 159         }
 160
 161         /* Hash continuation or HMAC, setup (inner) digest from state */
 162         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 163
 164         if (req->finish) {
 165                 /* Compute digest count for hash/HMAC finish operations */
 166                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 167                     req->hmac_zlen || (req->processed != req->block_sz)) {
 168                         count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
 169
 170                         /* This is a hardware limitation, as the
 171                          * counter must fit into an u32. This represents
 172                          * a fairly big amount of input data, so we
 173                          * shouldn't see this.
 174                          */
 175                         if (unlikely(count & 0xffffffff00000000ULL)) {
 176                                 dev_warn(priv->dev,
 177                                          "Input data is too big\n");
 178                                 return;
 179                         }
 180                 }
 181
 182                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 183                     /* Special case: zero length HMAC */
 184                     req->hmac_zlen ||
 185                     /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
 186                     (req->processed != req->block_sz)) {
 187                         /* Basic hash continue operation, need digest + cnt */
 188                         cdesc->control_data.control0 |=
 189                                 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
 190                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 191                                 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 192                         /* For zero-len HMAC, don't finalize, already padded! */
 193                         if (req->hmac_zlen)
 194                                 cdesc->control_data.control0 |=
 195                                         CONTEXT_CONTROL_NO_FINISH_HASH;
 196                         cdesc->control_data.control1 |=
 197                                 CONTEXT_CONTROL_DIGEST_CNT;
 198                         ctx->base.ctxr->data[req->state_sz >> 2] =
 199                                 cpu_to_le32(count);
 200                         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 201
 202                         /* Clear zero-length HMAC flag for next operation! */
 203                         req->hmac_zlen = false;
 204                 } else { /* HMAC */
 205                         /* Need outer digest for HMAC finalization */
 206                         memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
 207                                &ctx->base.opad, req->state_sz);
 208
 209                         /* Single pass HMAC - no digest count */
 210                         cdesc->control_data.control0 |=
 211                                 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
 212                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 213                                 CONTEXT_CONTROL_DIGEST_HMAC;
 214                 }
 215         } else { /* Hash continuation, do not finish yet */
 216                 cdesc->control_data.control0 |=
 217                         CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
 218                         CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
 219                         CONTEXT_CONTROL_TYPE_HASH_OUT |
 220                         CONTEXT_CONTROL_NO_FINISH_HASH;
 221         }
 222 }
 223
 224 static int safexcel_ahash_enqueue(struct ahash_request *areq);
 225
 226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
 227                                       int ring,
 228                                       struct crypto_async_request *async,
 229                                       bool *should_complete, int *ret)
 230 {
 231         struct safexcel_result_desc *rdesc;
 232         struct ahash_request *areq = ahash_request_cast(async);
 233         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 234         struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
 235         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 236         u64 cache_len;
 237
 238         *ret = 0;
 239
 240         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 241         if (IS_ERR(rdesc)) {
 242                 dev_err(priv->dev,
 243                         "hash: result: could not retrieve the result descriptor\n");
 244                 *ret = PTR_ERR(rdesc);
 245         } else {
 246                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 247         }
 248
 249         safexcel_complete(priv, ring);
 250
 251         if (sreq->nents) {
 252                 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
 253                 sreq->nents = 0;
 254         }
 255
 256         if (sreq->result_dma) {
 257                 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
 258                                  DMA_FROM_DEVICE);
 259                 sreq->result_dma = 0;
 260         }
 261
 262         if (sreq->cache_dma) {
 263                 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
 264                                  DMA_TO_DEVICE);
 265                 sreq->cache_dma = 0;
 266                 sreq->cache_sz = 0;
 267         }
 268
 269         if (sreq->finish) {
 270                 if (sreq->hmac &&
 271                     (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
 272                         /* Faking HMAC using hash - need to do outer hash */
 273                         memcpy(sreq->cache, sreq->state,
 274                                crypto_ahash_digestsize(ahash));
 275
 276                         memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
 277
 278                         sreq->len = sreq->block_sz +
 279                                     crypto_ahash_digestsize(ahash);
 280                         sreq->processed = sreq->block_sz;
 281                         sreq->hmac = 0;
 282
 283                         if (priv->flags & EIP197_TRC_CACHE)
 284                                 ctx->base.needs_inv = true;
 285                         areq->nbytes = 0;
 286                         safexcel_ahash_enqueue(areq);
 287
 288                         *should_complete = false; /* Not done yet */
 289                         return 1;
 290                 }
 291
 292                 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 293                              ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
 294                         /* Undo final XOR with 0xffffffff ...*/
 295                         *(__le32 *)areq->result = ~sreq->state[0];
 296                 } else {
 297                         memcpy(areq->result, sreq->state,
 298                                crypto_ahash_digestsize(ahash));
 299                 }
 300         }
 301
 302         cache_len = safexcel_queued_len(sreq);
 303         if (cache_len)
 304                 memcpy(sreq->cache, sreq->cache_next, cache_len);
 305
 306         *should_complete = true;
 307
 308         return 1;
 309 }
 310
 311 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 312                                    int *commands, int *results)
 313 {
 314         struct ahash_request *areq = ahash_request_cast(async);
 315         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 316         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 317         struct safexcel_crypto_priv *priv = ctx->base.priv;
 318         struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 319         struct safexcel_result_desc *rdesc;
 320         struct scatterlist *sg;
 321         struct safexcel_token *dmmy;
 322         int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
 323         u64 queued, len;
 324
 325         queued = safexcel_queued_len(req);
 326         if (queued <= HASH_CACHE_SIZE)
 327                 cache_len = queued;
 328         else
 329                 cache_len = queued - areq->nbytes;
 330
 331         if (!req->finish && !req->last_req) {
 332                 /* If this is not the last request and the queued data does not
 333                  * fit into full cache blocks, cache it for the next send call.
 334                  */
 335                 extra = queued & (HASH_CACHE_SIZE - 1);
 336
 337                 /* If this is not the last request and the queued data
 338                  * is a multiple of a block, cache the last one for now.
 339                  */
 340                 if (!extra)
 341                         extra = HASH_CACHE_SIZE;
 342
 343                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 344                                    req->cache_next, extra,
 345                                    areq->nbytes - extra);
 346
 347                 queued -= extra;
 348
 349                 if (!queued) {
 350                         *commands = 0;
 351                         *results = 0;
 352                         return 0;
 353                 }
 354
 355                 extra = 0;
 356         }
 357
 358         if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
 359                 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
 360                         /*
 361                          * Cache contains less than 1 full block, complete.
 362                          */
 363                         extra = AES_BLOCK_SIZE - cache_len;
 364                         if (queued > cache_len) {
 365                                 /* More data follows: borrow bytes */
 366                                 u64 tmp = queued - cache_len;
 367
 368                                 skip = min_t(u64, tmp, extra);
 369                                 sg_pcopy_to_buffer(areq->src,
 370                                         sg_nents(areq->src),
 371                                         req->cache + cache_len,
 372                                         skip, 0);
 373                         }
 374                         extra -= skip;
 375                         memset(req->cache + cache_len + skip, 0, extra);
 376                         if (!ctx->cbcmac && extra) {
 377                                 // 10- padding for XCBCMAC & CMAC
 378                                 req->cache[cache_len + skip] = 0x80;
 379                                 // HW will use K2 iso K3 - compensate!
 380                                 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
 381                                         u32 *cache = (void *)req->cache;
 382                                         u32 *ipad = ctx->base.ipad.word;
 383                                         u32 x;
 384
 385                                         x = ipad[i] ^ ipad[i + 4];
 386                                         cache[i] ^= swab(x);
 387                                 }
 388                         }
 389                         cache_len = AES_BLOCK_SIZE;
 390                         queued = queued + extra;
 391                 }
 392
 393                 /* XCBC continue: XOR previous result into 1st word */
 394                 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
 395         }
 396
 397         len = queued;
 398         /* Add a command descriptor for the cached data, if any */
 399         if (cache_len) {
 400                 req->cache_dma = dma_map_single(priv->dev, req->cache,
 401                                                 cache_len, DMA_TO_DEVICE);
 402                 if (dma_mapping_error(priv->dev, req->cache_dma))
 403                         return -EINVAL;
 404
 405                 req->cache_sz = cache_len;
 406                 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
 407                                                  (cache_len == len),
 408                                                  req->cache_dma, cache_len,
 409                                                  len, ctx->base.ctxr_dma,
 410                                                  &dmmy);
 411                 if (IS_ERR(first_cdesc)) {
 412                         ret = PTR_ERR(first_cdesc);
 413                         goto unmap_cache;
 414                 }
 415                 n_cdesc++;
 416
 417                 queued -= cache_len;
 418                 if (!queued)
 419                         goto send_command;
 420         }
 421
 422         /* Now handle the current ahash request buffer(s) */
 423         req->nents = dma_map_sg(priv->dev, areq->src,
 424                                 sg_nents_for_len(areq->src,
 425                                                  areq->nbytes),
 426                                 DMA_TO_DEVICE);
 427         if (!req->nents) {
 428                 ret = -ENOMEM;
 429                 goto cdesc_rollback;
 430         }
 431
 432         for_each_sg(areq->src, sg, req->nents, i) {
 433                 int sglen = sg_dma_len(sg);
 434
 435                 if (unlikely(sglen <= skip)) {
 436                         skip -= sglen;
 437                         continue;
 438                 }
 439
 440                 /* Do not overflow the request */
 441                 if ((queued + skip) <= sglen)
 442                         sglen = queued;
 443                 else
 444                         sglen -= skip;
 445
 446                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 447                                            !(queued - sglen),
 448                                            sg_dma_address(sg) + skip, sglen,
 449                                            len, ctx->base.ctxr_dma, &dmmy);
 450                 if (IS_ERR(cdesc)) {
 451                         ret = PTR_ERR(cdesc);
 452                         goto unmap_sg;
 453                 }
 454
 455                 if (!n_cdesc)
 456                         first_cdesc = cdesc;
 457                 n_cdesc++;
 458
 459                 queued -= sglen;
 460                 if (!queued)
 461                         break;
 462                 skip = 0;
 463         }
 464
 465 send_command:
 466         /* Setup the context options */
 467         safexcel_context_control(ctx, req, first_cdesc);
 468
 469         /* Add the token */
 470         safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
 471
 472         req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
 473                                          DMA_FROM_DEVICE);
 474         if (dma_mapping_error(priv->dev, req->result_dma)) {
 475                 ret = -EINVAL;
 476                 goto unmap_sg;
 477         }
 478
 479         /* Add a result descriptor */
 480         rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
 481                                    req->digest_sz);
 482         if (IS_ERR(rdesc)) {
 483                 ret = PTR_ERR(rdesc);
 484                 goto unmap_result;
 485         }
 486
 487         safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
 488
 489         req->processed += len - extra;
 490
 491         *commands = n_cdesc;
 492         *results = 1;
 493         return 0;
 494
 495 unmap_result:
 496         dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
 497                          DMA_FROM_DEVICE);
 498 unmap_sg:
 499         if (req->nents) {
 500                 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
 501                 req->nents = 0;
 502         }
 503 cdesc_rollback:
 504         for (i = 0; i < n_cdesc; i++)
 505                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 506 unmap_cache:
 507         if (req->cache_dma) {
 508                 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
 509                                  DMA_TO_DEVICE);
 510                 req->cache_dma = 0;
 511                 req->cache_sz = 0;
 512         }
 513
 514         return ret;
 515 }
 516
 517 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 518                                       int ring,
 519                                       struct crypto_async_request *async,
 520                                       bool *should_complete, int *ret)
 521 {
 522         struct safexcel_result_desc *rdesc;
 523         struct ahash_request *areq = ahash_request_cast(async);
 524         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 525         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 526         int enq_ret;
 527
 528         *ret = 0;
 529
 530         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 531         if (IS_ERR(rdesc)) {
 532                 dev_err(priv->dev,
 533                         "hash: invalidate: could not retrieve the result descriptor\n");
 534                 *ret = PTR_ERR(rdesc);
 535         } else {
 536                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 537         }
 538
 539         safexcel_complete(priv, ring);
 540
 541         if (ctx->base.exit_inv) {
 542                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 543                               ctx->base.ctxr_dma);
 544
 545                 *should_complete = true;
 546                 return 1;
 547         }
 548
 549         ring = safexcel_select_ring(priv);
 550         ctx->base.ring = ring;
 551
 552         spin_lock_bh(&priv->ring[ring].queue_lock);
 553         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
 554         spin_unlock_bh(&priv->ring[ring].queue_lock);
 555
 556         if (enq_ret != -EINPROGRESS)
 557                 *ret = enq_ret;
 558
 559         queue_work(priv->ring[ring].workqueue,
 560                    &priv->ring[ring].work_data.work);
 561
 562         *should_complete = false;
 563
 564         return 1;
 565 }
 566
 567 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 568                                   struct crypto_async_request *async,
 569                                   bool *should_complete, int *ret)
 570 {
 571         struct ahash_request *areq = ahash_request_cast(async);
 572         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 573         int err;
 574
 575         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
 576
 577         if (req->needs_inv) {
 578                 req->needs_inv = false;
 579                 err = safexcel_handle_inv_result(priv, ring, async,
 580                                                  should_complete, ret);
 581         } else {
 582                 err = safexcel_handle_req_result(priv, ring, async,
 583                                                  should_complete, ret);
 584         }
 585
 586         return err;
 587 }
 588
 589 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 590                                    int ring, int *commands, int *results)
 591 {
 592         struct ahash_request *areq = ahash_request_cast(async);
 593         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 594         int ret;
 595
 596         ret = safexcel_invalidate_cache(async, ctx->base.priv,
 597                                         ctx->base.ctxr_dma, ring);
 598         if (unlikely(ret))
 599                 return ret;
 600
 601         *commands = 1;
 602         *results = 1;
 603
 604         return 0;
 605 }
 606
 607 static int safexcel_ahash_send(struct crypto_async_request *async,
 608                                int ring, int *commands, int *results)
 609 {
 610         struct ahash_request *areq = ahash_request_cast(async);
 611         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 612         int ret;
 613
 614         if (req->needs_inv)
 615                 ret = safexcel_ahash_send_inv(async, ring, commands, results);
 616         else
 617                 ret = safexcel_ahash_send_req(async, ring, commands, results);
 618
 619         return ret;
 620 }
 621
 622 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 623 {
 624         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 625         struct safexcel_crypto_priv *priv = ctx->base.priv;
 626         EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
 627         struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
 628         struct safexcel_inv_result result = {};
 629         int ring = ctx->base.ring;
 630
 631         memset(req, 0, EIP197_AHASH_REQ_SIZE);
 632
 633         /* create invalidation request */
 634         init_completion(&result.completion);
 635         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 636                                    safexcel_inv_complete, &result);
 637
 638         ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
 639         ctx = crypto_tfm_ctx(req->base.tfm);
 640         ctx->base.exit_inv = true;
 641         rctx->needs_inv = true;
 642
 643         spin_lock_bh(&priv->ring[ring].queue_lock);
 644         crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 645         spin_unlock_bh(&priv->ring[ring].queue_lock);
 646
 647         queue_work(priv->ring[ring].workqueue,
 648                    &priv->ring[ring].work_data.work);
 649
 650         wait_for_completion(&result.completion);
 651
 652         if (result.error) {
 653                 dev_warn(priv->dev, "hash: completion error (%d)\n",
 654                          result.error);
 655                 return result.error;
 656         }
 657
 658         return 0;
 659 }
 660
 661 /* safexcel_ahash_cache: cache data until at least one request can be sent to
 662  * the engine, aka. when there is at least 1 block size in the pipe.
 663  */
 664 static int safexcel_ahash_cache(struct ahash_request *areq)
 665 {
 666         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 667         u64 cache_len;
 668
 669         /* cache_len: everything accepted by the driver but not sent yet,
 670          * tot sz handled by update() - last req sz - tot sz handled by send()
 671          */
 672         cache_len = safexcel_queued_len(req);
 673
 674         /*
 675          * In case there isn't enough bytes to proceed (less than a
 676          * block size), cache the data until we have enough.
 677          */
 678         if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
 679                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 680                                    req->cache + cache_len,
 681                                    areq->nbytes, 0);
 682                 return 0;
 683         }
 684
 685         /* We couldn't cache all the data */
 686         return -E2BIG;
 687 }
 688
 689 static int safexcel_ahash_enqueue(struct ahash_request *areq)
 690 {
 691         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 692         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 693         struct safexcel_crypto_priv *priv = ctx->base.priv;
 694         int ret, ring;
 695
 696         req->needs_inv = false;
 697
 698         if (ctx->base.ctxr) {
 699                 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
 700                      /* invalidate for *any* non-XCBC continuation */
 701                    ((req->not_first && !req->xcbcmac) ||
 702                      /* invalidate if (i)digest changed */
 703                      memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
 704                      /* invalidate for HMAC finish with odigest changed */
 705                      (req->finish && req->hmac &&
 706                       memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
 707                              &ctx->base.opad, req->state_sz))))
 708                         /*
 709                          * We're still setting needs_inv here, even though it is
 710                          * cleared right away, because the needs_inv flag can be
 711                          * set in other functions and we want to keep the same
 712                          * logic.
 713                          */
 714                         ctx->base.needs_inv = true;
 715
 716                 if (ctx->base.needs_inv) {
 717                         ctx->base.needs_inv = false;
 718                         req->needs_inv = true;
 719                 }
 720         } else {
 721                 ctx->base.ring = safexcel_select_ring(priv);
 722                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 723                                                  EIP197_GFP_FLAGS(areq->base),
 724                                                  &ctx->base.ctxr_dma);
 725                 if (!ctx->base.ctxr)
 726                         return -ENOMEM;
 727         }
 728         req->not_first = true;
 729
 730         ring = ctx->base.ring;
 731
 732         spin_lock_bh(&priv->ring[ring].queue_lock);
 733         ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 734         spin_unlock_bh(&priv->ring[ring].queue_lock);
 735
 736         queue_work(priv->ring[ring].workqueue,
 737                    &priv->ring[ring].work_data.work);
 738
 739         return ret;
 740 }
 741
 742 static int safexcel_ahash_update(struct ahash_request *areq)
 743 {
 744         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 745         int ret;
 746
 747         /* If the request is 0 length, do nothing */
 748         if (!areq->nbytes)
 749                 return 0;
 750
 751         /* Add request to the cache if it fits */
 752         ret = safexcel_ahash_cache(areq);
 753
 754         /* Update total request length */
 755         req->len += areq->nbytes;
 756
 757         /* If not all data could fit into the cache, go process the excess.
 758          * Also go process immediately for an HMAC IV precompute, which
 759          * will never be finished at all, but needs to be processed anyway.
 760          */
 761         if ((ret && !req->finish) || req->last_req)
 762                 return safexcel_ahash_enqueue(areq);
 763
 764         return 0;
 765 }
 766
 767 static int safexcel_ahash_final(struct ahash_request *areq)
 768 {
 769         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 770         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 771
 772         req->finish = true;
 773
 774         if (unlikely(!req->len && !areq->nbytes)) {
 775                 /*
 776                  * If we have an overall 0 length *hash* request:
 777                  * The HW cannot do 0 length hash, so we provide the correct
 778                  * result directly here.
 779                  */
 780                 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
 781                         memcpy(areq->result, md5_zero_message_hash,
 782                                MD5_DIGEST_SIZE);
 783                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
 784                         memcpy(areq->result, sha1_zero_message_hash,
 785                                SHA1_DIGEST_SIZE);
 786                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
 787                         memcpy(areq->result, sha224_zero_message_hash,
 788                                SHA224_DIGEST_SIZE);
 789                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
 790                         memcpy(areq->result, sha256_zero_message_hash,
 791                                SHA256_DIGEST_SIZE);
 792                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
 793                         memcpy(areq->result, sha384_zero_message_hash,
 794                                SHA384_DIGEST_SIZE);
 795                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
 796                         memcpy(areq->result, sha512_zero_message_hash,
 797                                SHA512_DIGEST_SIZE);
 798                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
 799                         memcpy(areq->result,
 800                                EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
 801                 }
 802
 803                 return 0;
 804         } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 805                             ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
 806                             req->len == sizeof(u32) && !areq->nbytes)) {
 807                 /* Zero length CRC32 */
 808                 memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
 809                 return 0;
 810         } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
 811                             !areq->nbytes)) {
 812                 /* Zero length CBC MAC */
 813                 memset(areq->result, 0, AES_BLOCK_SIZE);
 814                 return 0;
 815         } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
 816                             !areq->nbytes)) {
 817                 /* Zero length (X)CBC/CMAC */
 818                 int i;
 819
 820                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
 821                         u32 *result = (void *)areq->result;
 822
 823                         /* K3 */
 824                         result[i] = swab(ctx->base.ipad.word[i + 4]);
 825                 }
 826                 areq->result[0] ^= 0x80;                        // 10- padding
 827                 crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
 828                 return 0;
 829         } else if (unlikely(req->hmac &&
 830                             (req->len == req->block_sz) &&
 831                             !areq->nbytes)) {
 832                 /*
 833                  * If we have an overall 0 length *HMAC* request:
 834                  * For HMAC, we need to finalize the inner digest
 835                  * and then perform the outer hash.
 836                  */
 837
 838                 /* generate pad block in the cache */
 839                 /* start with a hash block of all zeroes */
 840                 memset(req->cache, 0, req->block_sz);
 841                 /* set the first byte to 0x80 to 'append a 1 bit' */
 842                 req->cache[0] = 0x80;
 843                 /* add the length in bits in the last 2 bytes */
 844                 if (req->len_is_le) {
 845                         /* Little endian length word (e.g. MD5) */
 846                         req->cache[req->block_sz-8] = (req->block_sz << 3) &
 847                                                       255;
 848                         req->cache[req->block_sz-7] = (req->block_sz >> 5);
 849                 } else {
 850                         /* Big endian length word (e.g. any SHA) */
 851                         req->cache[req->block_sz-2] = (req->block_sz >> 5);
 852                         req->cache[req->block_sz-1] = (req->block_sz << 3) &
 853                                                       255;
 854                 }
 855
 856                 req->len += req->block_sz; /* plus 1 hash block */
 857
 858                 /* Set special zero-length HMAC flag */
 859                 req->hmac_zlen = true;
 860
 861                 /* Finalize HMAC */
 862                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 863         } else if (req->hmac) {
 864                 /* Finalize HMAC */
 865                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 866         }
 867
 868         return safexcel_ahash_enqueue(areq);
 869 }
 870
 871 static int safexcel_ahash_finup(struct ahash_request *areq)
 872 {
 873         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 874
 875         req->finish = true;
 876
 877         safexcel_ahash_update(areq);
 878         return safexcel_ahash_final(areq);
 879 }
 880
 881 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 882 {
 883         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 884         struct safexcel_ahash_export_state *export = out;
 885
 886         export->len = req->len;
 887         export->processed = req->processed;
 888
 889         export->digest = req->digest;
 890
 891         memcpy(export->state, req->state, req->state_sz);
 892         memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
 893
 894         return 0;
 895 }
 896
 897 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 898 {
 899         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 900         const struct safexcel_ahash_export_state *export = in;
 901         int ret;
 902
 903         ret = crypto_ahash_init(areq);
 904         if (ret)
 905                 return ret;
 906
 907         req->len = export->len;
 908         req->processed = export->processed;
 909
 910         req->digest = export->digest;
 911
 912         memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
 913         memcpy(req->state, export->state, req->state_sz);
 914
 915         return 0;
 916 }
 917
 918 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
 919 {
 920         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 921         struct safexcel_alg_template *tmpl =
 922                 container_of(__crypto_ahash_alg(tfm->__crt_alg),
 923                              struct safexcel_alg_template, alg.ahash);
 924
 925         ctx->base.priv = tmpl->priv;
 926         ctx->base.send = safexcel_ahash_send;
 927         ctx->base.handle_result = safexcel_handle_result;
 928         ctx->fb_do_setkey = false;
 929
 930         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 931                                  sizeof(struct safexcel_ahash_req));
 932         return 0;
 933 }
 934
 935 static int safexcel_sha1_init(struct ahash_request *areq)
 936 {
 937         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 938         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 939
 940         memset(req, 0, sizeof(*req));
 941
 942         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
 943         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 944         req->state_sz = SHA1_DIGEST_SIZE;
 945         req->digest_sz = SHA1_DIGEST_SIZE;
 946         req->block_sz = SHA1_BLOCK_SIZE;
 947
 948         return 0;
 949 }
 950
 951 static int safexcel_sha1_digest(struct ahash_request *areq)
 952 {
 953         int ret = safexcel_sha1_init(areq);
 954
 955         if (ret)
 956                 return ret;
 957
 958         return safexcel_ahash_finup(areq);
 959 }
 960
 961 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 962 {
 963         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 964         struct safexcel_crypto_priv *priv = ctx->base.priv;
 965         int ret;
 966
 967         /* context not allocated, skip invalidation */
 968         if (!ctx->base.ctxr)
 969                 return;
 970
 971         if (priv->flags & EIP197_TRC_CACHE) {
 972                 ret = safexcel_ahash_exit_inv(tfm);
 973                 if (ret)
 974                         dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
 975         } else {
 976                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 977                               ctx->base.ctxr_dma);
 978         }
 979 }
 980
 981 struct safexcel_alg_template safexcel_alg_sha1 = {
 982         .type = SAFEXCEL_ALG_TYPE_AHASH,
 983         .algo_mask = SAFEXCEL_ALG_SHA1,
 984         .alg.ahash = {
 985                 .init = safexcel_sha1_init,
 986                 .update = safexcel_ahash_update,
 987                 .final = safexcel_ahash_final,
 988                 .finup = safexcel_ahash_finup,
 989                 .digest = safexcel_sha1_digest,
 990                 .export = safexcel_ahash_export,
 991                 .import = safexcel_ahash_import,
 992                 .halg = {
 993                         .digestsize = SHA1_DIGEST_SIZE,
 994                         .statesize = sizeof(struct safexcel_ahash_export_state),
 995                         .base = {
 996                                 .cra_name = "sha1",
 997                                 .cra_driver_name = "safexcel-sha1",
 998                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
 999                                 .cra_flags = CRYPTO_ALG_ASYNC |
1000                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1001                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1002                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1003                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1004                                 .cra_init = safexcel_ahash_cra_init,
1005                                 .cra_exit = safexcel_ahash_cra_exit,
1006                                 .cra_module = THIS_MODULE,
1007                         },
1008                 },
1009         },
1010 };
1011
1012 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1013 {
1014         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1015         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1016
1017         memset(req, 0, sizeof(*req));
1018
1019         /* Start from ipad precompute */
1020         memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1021         /* Already processed the key^ipad part now! */
1022         req->len        = SHA1_BLOCK_SIZE;
1023         req->processed  = SHA1_BLOCK_SIZE;
1024
1025         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1026         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1027         req->state_sz = SHA1_DIGEST_SIZE;
1028         req->digest_sz = SHA1_DIGEST_SIZE;
1029         req->block_sz = SHA1_BLOCK_SIZE;
1030         req->hmac = true;
1031
1032         return 0;
1033 }
1034
1035 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1036 {
1037         int ret = safexcel_hmac_sha1_init(areq);
1038
1039         if (ret)
1040                 return ret;
1041
1042         return safexcel_ahash_finup(areq);
1043 }
1044
1045 struct safexcel_ahash_result {
1046         struct completion completion;
1047         int error;
1048 };
1049
1050 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1051 {
1052         struct safexcel_ahash_result *result = req->data;
1053
1054         if (error == -EINPROGRESS)
1055                 return;
1056
1057         result->error = error;
1058         complete(&result->completion);
1059 }
1060
1061 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1062                                   unsigned int blocksize, const u8 *key,
1063                                   unsigned int keylen, u8 *ipad, u8 *opad)
1064 {
1065         struct safexcel_ahash_result result;
1066         struct scatterlist sg;
1067         int ret, i;
1068         u8 *keydup;
1069
1070         if (keylen <= blocksize) {
1071                 memcpy(ipad, key, keylen);
1072         } else {
1073                 keydup = kmemdup(key, keylen, GFP_KERNEL);
1074                 if (!keydup)
1075                         return -ENOMEM;
1076
1077                 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1078                                            safexcel_ahash_complete, &result);
1079                 sg_init_one(&sg, keydup, keylen);
1080                 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1081                 init_completion(&result.completion);
1082
1083                 ret = crypto_ahash_digest(areq);
1084                 if (ret == -EINPROGRESS || ret == -EBUSY) {
1085                         wait_for_completion_interruptible(&result.completion);
1086                         ret = result.error;
1087                 }
1088
1089                 /* Avoid leaking */
1090                 kfree_sensitive(keydup);
1091
1092                 if (ret)
1093                         return ret;
1094
1095                 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1096         }
1097
1098         memset(ipad + keylen, 0, blocksize - keylen);
1099         memcpy(opad, ipad, blocksize);
1100
1101         for (i = 0; i < blocksize; i++) {
1102                 ipad[i] ^= HMAC_IPAD_VALUE;
1103                 opad[i] ^= HMAC_OPAD_VALUE;
1104         }
1105
1106         return 0;
1107 }
1108
1109 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1110                                  unsigned int blocksize, u8 *pad, void *state)
1111 {
1112         struct safexcel_ahash_result result;
1113         struct safexcel_ahash_req *req;
1114         struct scatterlist sg;
1115         int ret;
1116
1117         ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1118                                    safexcel_ahash_complete, &result);
1119         sg_init_one(&sg, pad, blocksize);
1120         ahash_request_set_crypt(areq, &sg, pad, blocksize);
1121         init_completion(&result.completion);
1122
1123         ret = crypto_ahash_init(areq);
1124         if (ret)
1125                 return ret;
1126
1127         req = ahash_request_ctx(areq);
1128         req->hmac = true;
1129         req->last_req = true;
1130
1131         ret = crypto_ahash_update(areq);
1132         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1133                 return ret;
1134
1135         wait_for_completion_interruptible(&result.completion);
1136         if (result.error)
1137                 return result.error;
1138
1139         return crypto_ahash_export(areq, state);
1140 }
1141
1142 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1143                                   unsigned int keylen,
1144                                   void *istate, void *ostate)
1145 {
1146         struct ahash_request *areq;
1147         struct crypto_ahash *tfm;
1148         unsigned int blocksize;
1149         u8 *ipad, *opad;
1150         int ret;
1151
1152         tfm = crypto_alloc_ahash(alg, 0, 0);
1153         if (IS_ERR(tfm))
1154                 return PTR_ERR(tfm);
1155
1156         areq = ahash_request_alloc(tfm, GFP_KERNEL);
1157         if (!areq) {
1158                 ret = -ENOMEM;
1159                 goto free_ahash;
1160         }
1161
1162         crypto_ahash_clear_flags(tfm, ~0);
1163         blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1164
1165         ipad = kcalloc(2, blocksize, GFP_KERNEL);
1166         if (!ipad) {
1167                 ret = -ENOMEM;
1168                 goto free_request;
1169         }
1170
1171         opad = ipad + blocksize;
1172
1173         ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1174         if (ret)
1175                 goto free_ipad;
1176
1177         ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1178         if (ret)
1179                 goto free_ipad;
1180
1181         ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1182
1183 free_ipad:
1184         kfree(ipad);
1185 free_request:
1186         ahash_request_free(areq);
1187 free_ahash:
1188         crypto_free_ahash(tfm);
1189
1190         return ret;
1191 }
1192
1193 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1194                          unsigned int keylen, const char *alg,
1195                          unsigned int state_sz)
1196 {
1197         struct safexcel_crypto_priv *priv = base->priv;
1198         struct safexcel_ahash_export_state istate, ostate;
1199         int ret;
1200
1201         ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1202         if (ret)
1203                 return ret;
1204
1205         if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1206             (memcmp(&base->ipad, istate.state, state_sz) ||
1207              memcmp(&base->opad, ostate.state, state_sz)))
1208                 base->needs_inv = true;
1209
1210         memcpy(&base->ipad, &istate.state, state_sz);
1211         memcpy(&base->opad, &ostate.state, state_sz);
1212
1213         return 0;
1214 }
1215
1216 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1217                                     unsigned int keylen, const char *alg,
1218                                     unsigned int state_sz)
1219 {
1220         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1221
1222         return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1223 }
1224
1225 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1226                                      unsigned int keylen)
1227 {
1228         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1229                                         SHA1_DIGEST_SIZE);
1230 }
1231
1232 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1233         .type = SAFEXCEL_ALG_TYPE_AHASH,
1234         .algo_mask = SAFEXCEL_ALG_SHA1,
1235         .alg.ahash = {
1236                 .init = safexcel_hmac_sha1_init,
1237                 .update = safexcel_ahash_update,
1238                 .final = safexcel_ahash_final,
1239                 .finup = safexcel_ahash_finup,
1240                 .digest = safexcel_hmac_sha1_digest,
1241                 .setkey = safexcel_hmac_sha1_setkey,
1242                 .export = safexcel_ahash_export,
1243                 .import = safexcel_ahash_import,
1244                 .halg = {
1245                         .digestsize = SHA1_DIGEST_SIZE,
1246                         .statesize = sizeof(struct safexcel_ahash_export_state),
1247                         .base = {
1248                                 .cra_name = "hmac(sha1)",
1249                                 .cra_driver_name = "safexcel-hmac-sha1",
1250                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1251                                 .cra_flags = CRYPTO_ALG_ASYNC |
1252                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1253                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1254                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1255                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1256                                 .cra_init = safexcel_ahash_cra_init,
1257                                 .cra_exit = safexcel_ahash_cra_exit,
1258                                 .cra_module = THIS_MODULE,
1259                         },
1260                 },
1261         },
1262 };
1263
1264 static int safexcel_sha256_init(struct ahash_request *areq)
1265 {
1266         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1267         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1268
1269         memset(req, 0, sizeof(*req));
1270
1271         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1272         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1273         req->state_sz = SHA256_DIGEST_SIZE;
1274         req->digest_sz = SHA256_DIGEST_SIZE;
1275         req->block_sz = SHA256_BLOCK_SIZE;
1276
1277         return 0;
1278 }
1279
1280 static int safexcel_sha256_digest(struct ahash_request *areq)
1281 {
1282         int ret = safexcel_sha256_init(areq);
1283
1284         if (ret)
1285                 return ret;
1286
1287         return safexcel_ahash_finup(areq);
1288 }
1289
1290 struct safexcel_alg_template safexcel_alg_sha256 = {
1291         .type = SAFEXCEL_ALG_TYPE_AHASH,
1292         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1293         .alg.ahash = {
1294                 .init = safexcel_sha256_init,
1295                 .update = safexcel_ahash_update,
1296                 .final = safexcel_ahash_final,
1297                 .finup = safexcel_ahash_finup,
1298                 .digest = safexcel_sha256_digest,
1299                 .export = safexcel_ahash_export,
1300                 .import = safexcel_ahash_import,
1301                 .halg = {
1302                         .digestsize = SHA256_DIGEST_SIZE,
1303                         .statesize = sizeof(struct safexcel_ahash_export_state),
1304                         .base = {
1305                                 .cra_name = "sha256",
1306                                 .cra_driver_name = "safexcel-sha256",
1307                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1308                                 .cra_flags = CRYPTO_ALG_ASYNC |
1309                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1310                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1311                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1312                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1313                                 .cra_init = safexcel_ahash_cra_init,
1314                                 .cra_exit = safexcel_ahash_cra_exit,
1315                                 .cra_module = THIS_MODULE,
1316                         },
1317                 },
1318         },
1319 };
1320
1321 static int safexcel_sha224_init(struct ahash_request *areq)
1322 {
1323         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1324         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1325
1326         memset(req, 0, sizeof(*req));
1327
1328         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1329         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1330         req->state_sz = SHA256_DIGEST_SIZE;
1331         req->digest_sz = SHA256_DIGEST_SIZE;
1332         req->block_sz = SHA256_BLOCK_SIZE;
1333
1334         return 0;
1335 }
1336
1337 static int safexcel_sha224_digest(struct ahash_request *areq)
1338 {
1339         int ret = safexcel_sha224_init(areq);
1340
1341         if (ret)
1342                 return ret;
1343
1344         return safexcel_ahash_finup(areq);
1345 }
1346
1347 struct safexcel_alg_template safexcel_alg_sha224 = {
1348         .type = SAFEXCEL_ALG_TYPE_AHASH,
1349         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1350         .alg.ahash = {
1351                 .init = safexcel_sha224_init,
1352                 .update = safexcel_ahash_update,
1353                 .final = safexcel_ahash_final,
1354                 .finup = safexcel_ahash_finup,
1355                 .digest = safexcel_sha224_digest,
1356                 .export = safexcel_ahash_export,
1357                 .import = safexcel_ahash_import,
1358                 .halg = {
1359                         .digestsize = SHA224_DIGEST_SIZE,
1360                         .statesize = sizeof(struct safexcel_ahash_export_state),
1361                         .base = {
1362                                 .cra_name = "sha224",
1363                                 .cra_driver_name = "safexcel-sha224",
1364                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1365                                 .cra_flags = CRYPTO_ALG_ASYNC |
1366                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1367                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1368                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1369                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1370                                 .cra_init = safexcel_ahash_cra_init,
1371                                 .cra_exit = safexcel_ahash_cra_exit,
1372                                 .cra_module = THIS_MODULE,
1373                         },
1374                 },
1375         },
1376 };
1377
1378 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1379                                        unsigned int keylen)
1380 {
1381         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1382                                         SHA256_DIGEST_SIZE);
1383 }
1384
1385 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1386 {
1387         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1388         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1389
1390         memset(req, 0, sizeof(*req));
1391
1392         /* Start from ipad precompute */
1393         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1394         /* Already processed the key^ipad part now! */
1395         req->len        = SHA256_BLOCK_SIZE;
1396         req->processed  = SHA256_BLOCK_SIZE;
1397
1398         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1399         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1400         req->state_sz = SHA256_DIGEST_SIZE;
1401         req->digest_sz = SHA256_DIGEST_SIZE;
1402         req->block_sz = SHA256_BLOCK_SIZE;
1403         req->hmac = true;
1404
1405         return 0;
1406 }
1407
1408 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1409 {
1410         int ret = safexcel_hmac_sha224_init(areq);
1411
1412         if (ret)
1413                 return ret;
1414
1415         return safexcel_ahash_finup(areq);
1416 }
1417
1418 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1419         .type = SAFEXCEL_ALG_TYPE_AHASH,
1420         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1421         .alg.ahash = {
1422                 .init = safexcel_hmac_sha224_init,
1423                 .update = safexcel_ahash_update,
1424                 .final = safexcel_ahash_final,
1425                 .finup = safexcel_ahash_finup,
1426                 .digest = safexcel_hmac_sha224_digest,
1427                 .setkey = safexcel_hmac_sha224_setkey,
1428                 .export = safexcel_ahash_export,
1429                 .import = safexcel_ahash_import,
1430                 .halg = {
1431                         .digestsize = SHA224_DIGEST_SIZE,
1432                         .statesize = sizeof(struct safexcel_ahash_export_state),
1433                         .base = {
1434                                 .cra_name = "hmac(sha224)",
1435                                 .cra_driver_name = "safexcel-hmac-sha224",
1436                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1437                                 .cra_flags = CRYPTO_ALG_ASYNC |
1438                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1439                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1440                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1441                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1442                                 .cra_init = safexcel_ahash_cra_init,
1443                                 .cra_exit = safexcel_ahash_cra_exit,
1444                                 .cra_module = THIS_MODULE,
1445                         },
1446                 },
1447         },
1448 };
1449
1450 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1451                                      unsigned int keylen)
1452 {
1453         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1454                                         SHA256_DIGEST_SIZE);
1455 }
1456
1457 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1458 {
1459         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1460         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1461
1462         memset(req, 0, sizeof(*req));
1463
1464         /* Start from ipad precompute */
1465         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1466         /* Already processed the key^ipad part now! */
1467         req->len        = SHA256_BLOCK_SIZE;
1468         req->processed  = SHA256_BLOCK_SIZE;
1469
1470         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1471         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1472         req->state_sz = SHA256_DIGEST_SIZE;
1473         req->digest_sz = SHA256_DIGEST_SIZE;
1474         req->block_sz = SHA256_BLOCK_SIZE;
1475         req->hmac = true;
1476
1477         return 0;
1478 }
1479
1480 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1481 {
1482         int ret = safexcel_hmac_sha256_init(areq);
1483
1484         if (ret)
1485                 return ret;
1486
1487         return safexcel_ahash_finup(areq);
1488 }
1489
1490 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1491         .type = SAFEXCEL_ALG_TYPE_AHASH,
1492         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1493         .alg.ahash = {
1494                 .init = safexcel_hmac_sha256_init,
1495                 .update = safexcel_ahash_update,
1496                 .final = safexcel_ahash_final,
1497                 .finup = safexcel_ahash_finup,
1498                 .digest = safexcel_hmac_sha256_digest,
1499                 .setkey = safexcel_hmac_sha256_setkey,
1500                 .export = safexcel_ahash_export,
1501                 .import = safexcel_ahash_import,
1502                 .halg = {
1503                         .digestsize = SHA256_DIGEST_SIZE,
1504                         .statesize = sizeof(struct safexcel_ahash_export_state),
1505                         .base = {
1506                                 .cra_name = "hmac(sha256)",
1507                                 .cra_driver_name = "safexcel-hmac-sha256",
1508                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1509                                 .cra_flags = CRYPTO_ALG_ASYNC |
1510                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1511                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1512                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1513                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1514                                 .cra_init = safexcel_ahash_cra_init,
1515                                 .cra_exit = safexcel_ahash_cra_exit,
1516                                 .cra_module = THIS_MODULE,
1517                         },
1518                 },
1519         },
1520 };
1521
1522 static int safexcel_sha512_init(struct ahash_request *areq)
1523 {
1524         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1525         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1526
1527         memset(req, 0, sizeof(*req));
1528
1529         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1530         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1531         req->state_sz = SHA512_DIGEST_SIZE;
1532         req->digest_sz = SHA512_DIGEST_SIZE;
1533         req->block_sz = SHA512_BLOCK_SIZE;
1534
1535         return 0;
1536 }
1537
1538 static int safexcel_sha512_digest(struct ahash_request *areq)
1539 {
1540         int ret = safexcel_sha512_init(areq);
1541
1542         if (ret)
1543                 return ret;
1544
1545         return safexcel_ahash_finup(areq);
1546 }
1547
1548 struct safexcel_alg_template safexcel_alg_sha512 = {
1549         .type = SAFEXCEL_ALG_TYPE_AHASH,
1550         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1551         .alg.ahash = {
1552                 .init = safexcel_sha512_init,
1553                 .update = safexcel_ahash_update,
1554                 .final = safexcel_ahash_final,
1555                 .finup = safexcel_ahash_finup,
1556                 .digest = safexcel_sha512_digest,
1557                 .export = safexcel_ahash_export,
1558                 .import = safexcel_ahash_import,
1559                 .halg = {
1560                         .digestsize = SHA512_DIGEST_SIZE,
1561                         .statesize = sizeof(struct safexcel_ahash_export_state),
1562                         .base = {
1563                                 .cra_name = "sha512",
1564                                 .cra_driver_name = "safexcel-sha512",
1565                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1566                                 .cra_flags = CRYPTO_ALG_ASYNC |
1567                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1568                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1569                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1570                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1571                                 .cra_init = safexcel_ahash_cra_init,
1572                                 .cra_exit = safexcel_ahash_cra_exit,
1573                                 .cra_module = THIS_MODULE,
1574                         },
1575                 },
1576         },
1577 };
1578
1579 static int safexcel_sha384_init(struct ahash_request *areq)
1580 {
1581         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1582         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1583
1584         memset(req, 0, sizeof(*req));
1585
1586         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1587         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1588         req->state_sz = SHA512_DIGEST_SIZE;
1589         req->digest_sz = SHA512_DIGEST_SIZE;
1590         req->block_sz = SHA512_BLOCK_SIZE;
1591
1592         return 0;
1593 }
1594
1595 static int safexcel_sha384_digest(struct ahash_request *areq)
1596 {
1597         int ret = safexcel_sha384_init(areq);
1598
1599         if (ret)
1600                 return ret;
1601
1602         return safexcel_ahash_finup(areq);
1603 }
1604
1605 struct safexcel_alg_template safexcel_alg_sha384 = {
1606         .type = SAFEXCEL_ALG_TYPE_AHASH,
1607         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1608         .alg.ahash = {
1609                 .init = safexcel_sha384_init,
1610                 .update = safexcel_ahash_update,
1611                 .final = safexcel_ahash_final,
1612                 .finup = safexcel_ahash_finup,
1613                 .digest = safexcel_sha384_digest,
1614                 .export = safexcel_ahash_export,
1615                 .import = safexcel_ahash_import,
1616                 .halg = {
1617                         .digestsize = SHA384_DIGEST_SIZE,
1618                         .statesize = sizeof(struct safexcel_ahash_export_state),
1619                         .base = {
1620                                 .cra_name = "sha384",
1621                                 .cra_driver_name = "safexcel-sha384",
1622                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1623                                 .cra_flags = CRYPTO_ALG_ASYNC |
1624                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1625                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1626                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1627                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1628                                 .cra_init = safexcel_ahash_cra_init,
1629                                 .cra_exit = safexcel_ahash_cra_exit,
1630                                 .cra_module = THIS_MODULE,
1631                         },
1632                 },
1633         },
1634 };
1635
1636 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1637                                        unsigned int keylen)
1638 {
1639         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1640                                         SHA512_DIGEST_SIZE);
1641 }
1642
1643 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1644 {
1645         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1646         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1647
1648         memset(req, 0, sizeof(*req));
1649
1650         /* Start from ipad precompute */
1651         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1652         /* Already processed the key^ipad part now! */
1653         req->len        = SHA512_BLOCK_SIZE;
1654         req->processed  = SHA512_BLOCK_SIZE;
1655
1656         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1657         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1658         req->state_sz = SHA512_DIGEST_SIZE;
1659         req->digest_sz = SHA512_DIGEST_SIZE;
1660         req->block_sz = SHA512_BLOCK_SIZE;
1661         req->hmac = true;
1662
1663         return 0;
1664 }
1665
1666 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1667 {
1668         int ret = safexcel_hmac_sha512_init(areq);
1669
1670         if (ret)
1671                 return ret;
1672
1673         return safexcel_ahash_finup(areq);
1674 }
1675
1676 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1677         .type = SAFEXCEL_ALG_TYPE_AHASH,
1678         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1679         .alg.ahash = {
1680                 .init = safexcel_hmac_sha512_init,
1681                 .update = safexcel_ahash_update,
1682                 .final = safexcel_ahash_final,
1683                 .finup = safexcel_ahash_finup,
1684                 .digest = safexcel_hmac_sha512_digest,
1685                 .setkey = safexcel_hmac_sha512_setkey,
1686                 .export = safexcel_ahash_export,
1687                 .import = safexcel_ahash_import,
1688                 .halg = {
1689                         .digestsize = SHA512_DIGEST_SIZE,
1690                         .statesize = sizeof(struct safexcel_ahash_export_state),
1691                         .base = {
1692                                 .cra_name = "hmac(sha512)",
1693                                 .cra_driver_name = "safexcel-hmac-sha512",
1694                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1695                                 .cra_flags = CRYPTO_ALG_ASYNC |
1696                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1697                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1698                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1699                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1700                                 .cra_init = safexcel_ahash_cra_init,
1701                                 .cra_exit = safexcel_ahash_cra_exit,
1702                                 .cra_module = THIS_MODULE,
1703                         },
1704                 },
1705         },
1706 };
1707
1708 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1709                                        unsigned int keylen)
1710 {
1711         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1712                                         SHA512_DIGEST_SIZE);
1713 }
1714
1715 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1716 {
1717         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1718         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1719
1720         memset(req, 0, sizeof(*req));
1721
1722         /* Start from ipad precompute */
1723         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1724         /* Already processed the key^ipad part now! */
1725         req->len        = SHA512_BLOCK_SIZE;
1726         req->processed  = SHA512_BLOCK_SIZE;
1727
1728         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1729         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1730         req->state_sz = SHA512_DIGEST_SIZE;
1731         req->digest_sz = SHA512_DIGEST_SIZE;
1732         req->block_sz = SHA512_BLOCK_SIZE;
1733         req->hmac = true;
1734
1735         return 0;
1736 }
1737
1738 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1739 {
1740         int ret = safexcel_hmac_sha384_init(areq);
1741
1742         if (ret)
1743                 return ret;
1744
1745         return safexcel_ahash_finup(areq);
1746 }
1747
1748 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1749         .type = SAFEXCEL_ALG_TYPE_AHASH,
1750         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1751         .alg.ahash = {
1752                 .init = safexcel_hmac_sha384_init,
1753                 .update = safexcel_ahash_update,
1754                 .final = safexcel_ahash_final,
1755                 .finup = safexcel_ahash_finup,
1756                 .digest = safexcel_hmac_sha384_digest,
1757                 .setkey = safexcel_hmac_sha384_setkey,
1758                 .export = safexcel_ahash_export,
1759                 .import = safexcel_ahash_import,
1760                 .halg = {
1761                         .digestsize = SHA384_DIGEST_SIZE,
1762                         .statesize = sizeof(struct safexcel_ahash_export_state),
1763                         .base = {
1764                                 .cra_name = "hmac(sha384)",
1765                                 .cra_driver_name = "safexcel-hmac-sha384",
1766                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1767                                 .cra_flags = CRYPTO_ALG_ASYNC |
1768                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1769                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1770                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1771                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1772                                 .cra_init = safexcel_ahash_cra_init,
1773                                 .cra_exit = safexcel_ahash_cra_exit,
1774                                 .cra_module = THIS_MODULE,
1775                         },
1776                 },
1777         },
1778 };
1779
1780 static int safexcel_md5_init(struct ahash_request *areq)
1781 {
1782         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1783         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1784
1785         memset(req, 0, sizeof(*req));
1786
1787         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1788         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1789         req->state_sz = MD5_DIGEST_SIZE;
1790         req->digest_sz = MD5_DIGEST_SIZE;
1791         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1792
1793         return 0;
1794 }
1795
1796 static int safexcel_md5_digest(struct ahash_request *areq)
1797 {
1798         int ret = safexcel_md5_init(areq);
1799
1800         if (ret)
1801                 return ret;
1802
1803         return safexcel_ahash_finup(areq);
1804 }
1805
1806 struct safexcel_alg_template safexcel_alg_md5 = {
1807         .type = SAFEXCEL_ALG_TYPE_AHASH,
1808         .algo_mask = SAFEXCEL_ALG_MD5,
1809         .alg.ahash = {
1810                 .init = safexcel_md5_init,
1811                 .update = safexcel_ahash_update,
1812                 .final = safexcel_ahash_final,
1813                 .finup = safexcel_ahash_finup,
1814                 .digest = safexcel_md5_digest,
1815                 .export = safexcel_ahash_export,
1816                 .import = safexcel_ahash_import,
1817                 .halg = {
1818                         .digestsize = MD5_DIGEST_SIZE,
1819                         .statesize = sizeof(struct safexcel_ahash_export_state),
1820                         .base = {
1821                                 .cra_name = "md5",
1822                                 .cra_driver_name = "safexcel-md5",
1823                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1824                                 .cra_flags = CRYPTO_ALG_ASYNC |
1825                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1826                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1827                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1828                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1829                                 .cra_init = safexcel_ahash_cra_init,
1830                                 .cra_exit = safexcel_ahash_cra_exit,
1831                                 .cra_module = THIS_MODULE,
1832                         },
1833                 },
1834         },
1835 };
1836
1837 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1838 {
1839         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1840         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1841
1842         memset(req, 0, sizeof(*req));
1843
1844         /* Start from ipad precompute */
1845         memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1846         /* Already processed the key^ipad part now! */
1847         req->len        = MD5_HMAC_BLOCK_SIZE;
1848         req->processed  = MD5_HMAC_BLOCK_SIZE;
1849
1850         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1851         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1852         req->state_sz = MD5_DIGEST_SIZE;
1853         req->digest_sz = MD5_DIGEST_SIZE;
1854         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1855         req->len_is_le = true; /* MD5 is little endian! ... */
1856         req->hmac = true;
1857
1858         return 0;
1859 }
1860
1861 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1862                                      unsigned int keylen)
1863 {
1864         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1865                                         MD5_DIGEST_SIZE);
1866 }
1867
1868 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1869 {
1870         int ret = safexcel_hmac_md5_init(areq);
1871
1872         if (ret)
1873                 return ret;
1874
1875         return safexcel_ahash_finup(areq);
1876 }
1877
1878 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1879         .type = SAFEXCEL_ALG_TYPE_AHASH,
1880         .algo_mask = SAFEXCEL_ALG_MD5,
1881         .alg.ahash = {
1882                 .init = safexcel_hmac_md5_init,
1883                 .update = safexcel_ahash_update,
1884                 .final = safexcel_ahash_final,
1885                 .finup = safexcel_ahash_finup,
1886                 .digest = safexcel_hmac_md5_digest,
1887                 .setkey = safexcel_hmac_md5_setkey,
1888                 .export = safexcel_ahash_export,
1889                 .import = safexcel_ahash_import,
1890                 .halg = {
1891                         .digestsize = MD5_DIGEST_SIZE,
1892                         .statesize = sizeof(struct safexcel_ahash_export_state),
1893                         .base = {
1894                                 .cra_name = "hmac(md5)",
1895                                 .cra_driver_name = "safexcel-hmac-md5",
1896                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1897                                 .cra_flags = CRYPTO_ALG_ASYNC |
1898                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1899                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1900                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1901                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1902                                 .cra_init = safexcel_ahash_cra_init,
1903                                 .cra_exit = safexcel_ahash_cra_exit,
1904                                 .cra_module = THIS_MODULE,
1905                         },
1906                 },
1907         },
1908 };
1909
1910 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1911 {
1912         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1913         int ret = safexcel_ahash_cra_init(tfm);
1914
1915         /* Default 'key' is all zeroes */
1916         memset(&ctx->base.ipad, 0, sizeof(u32));
1917         return ret;
1918 }
1919
1920 static int safexcel_crc32_init(struct ahash_request *areq)
1921 {
1922         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1923         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1924
1925         memset(req, 0, sizeof(*req));
1926
1927         /* Start from loaded key */
1928         req->state[0]   = cpu_to_le32(~ctx->base.ipad.word[0]);
1929         /* Set processed to non-zero to enable invalidation detection */
1930         req->len        = sizeof(u32);
1931         req->processed  = sizeof(u32);
1932
1933         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1934         req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1935         req->state_sz = sizeof(u32);
1936         req->digest_sz = sizeof(u32);
1937         req->block_sz = sizeof(u32);
1938
1939         return 0;
1940 }
1941
1942 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1943                                  unsigned int keylen)
1944 {
1945         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1946
1947         if (keylen != sizeof(u32))
1948                 return -EINVAL;
1949
1950         memcpy(&ctx->base.ipad, key, sizeof(u32));
1951         return 0;
1952 }
1953
1954 static int safexcel_crc32_digest(struct ahash_request *areq)
1955 {
1956         return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1957 }
1958
1959 struct safexcel_alg_template safexcel_alg_crc32 = {
1960         .type = SAFEXCEL_ALG_TYPE_AHASH,
1961         .algo_mask = 0,
1962         .alg.ahash = {
1963                 .init = safexcel_crc32_init,
1964                 .update = safexcel_ahash_update,
1965                 .final = safexcel_ahash_final,
1966                 .finup = safexcel_ahash_finup,
1967                 .digest = safexcel_crc32_digest,
1968                 .setkey = safexcel_crc32_setkey,
1969                 .export = safexcel_ahash_export,
1970                 .import = safexcel_ahash_import,
1971                 .halg = {
1972                         .digestsize = sizeof(u32),
1973                         .statesize = sizeof(struct safexcel_ahash_export_state),
1974                         .base = {
1975                                 .cra_name = "crc32",
1976                                 .cra_driver_name = "safexcel-crc32",
1977                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1978                                 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1979                                              CRYPTO_ALG_ASYNC |
1980                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1981                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1982                                 .cra_blocksize = 1,
1983                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1984                                 .cra_init = safexcel_crc32_cra_init,
1985                                 .cra_exit = safexcel_ahash_cra_exit,
1986                                 .cra_module = THIS_MODULE,
1987                         },
1988                 },
1989         },
1990 };
1991
1992 static int safexcel_cbcmac_init(struct ahash_request *areq)
1993 {
1994         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1995         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1996
1997         memset(req, 0, sizeof(*req));
1998
1999         /* Start from loaded keys */
2000         memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
2001         /* Set processed to non-zero to enable invalidation detection */
2002         req->len        = AES_BLOCK_SIZE;
2003         req->processed  = AES_BLOCK_SIZE;
2004
2005         req->digest   = CONTEXT_CONTROL_DIGEST_XCM;
2006         req->state_sz = ctx->key_sz;
2007         req->digest_sz = AES_BLOCK_SIZE;
2008         req->block_sz = AES_BLOCK_SIZE;
2009         req->xcbcmac  = true;
2010
2011         return 0;
2012 }
2013
2014 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2015                                  unsigned int len)
2016 {
2017         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2018         struct crypto_aes_ctx aes;
2019         int ret, i;
2020
2021         ret = aes_expandkey(&aes, key, len);
2022         if (ret)
2023                 return ret;
2024
2025         memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2026         for (i = 0; i < len / sizeof(u32); i++)
2027                 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2028
2029         if (len == AES_KEYSIZE_192) {
2030                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2031                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2032         } else if (len == AES_KEYSIZE_256) {
2033                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2034                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2035         } else {
2036                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2037                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2038         }
2039         ctx->cbcmac  = true;
2040
2041         memzero_explicit(&aes, sizeof(aes));
2042         return 0;
2043 }
2044
2045 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2046 {
2047         return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2048 }
2049
2050 struct safexcel_alg_template safexcel_alg_cbcmac = {
2051         .type = SAFEXCEL_ALG_TYPE_AHASH,
2052         .algo_mask = 0,
2053         .alg.ahash = {
2054                 .init = safexcel_cbcmac_init,
2055                 .update = safexcel_ahash_update,
2056                 .final = safexcel_ahash_final,
2057                 .finup = safexcel_ahash_finup,
2058                 .digest = safexcel_cbcmac_digest,
2059                 .setkey = safexcel_cbcmac_setkey,
2060                 .export = safexcel_ahash_export,
2061                 .import = safexcel_ahash_import,
2062                 .halg = {
2063                         .digestsize = AES_BLOCK_SIZE,
2064                         .statesize = sizeof(struct safexcel_ahash_export_state),
2065                         .base = {
2066                                 .cra_name = "cbcmac(aes)",
2067                                 .cra_driver_name = "safexcel-cbcmac-aes",
2068                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2069                                 .cra_flags = CRYPTO_ALG_ASYNC |
2070                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2071                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2072                                 .cra_blocksize = 1,
2073                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2074                                 .cra_init = safexcel_ahash_cra_init,
2075                                 .cra_exit = safexcel_ahash_cra_exit,
2076                                 .cra_module = THIS_MODULE,
2077                         },
2078                 },
2079         },
2080 };
2081
2082 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2083                                  unsigned int len)
2084 {
2085         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2086         struct crypto_aes_ctx aes;
2087         u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2088         int ret, i;
2089
2090         ret = aes_expandkey(&aes, key, len);
2091         if (ret)
2092                 return ret;
2093
2094         /* precompute the XCBC key material */
2095         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2096         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2097                                 CRYPTO_TFM_REQ_MASK);
2098         ret = crypto_cipher_setkey(ctx->kaes, key, len);
2099         if (ret)
2100                 return ret;
2101
2102         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2103                 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2104         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp,
2105                 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2106         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2107                 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2108         for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2109                 ctx->base.ipad.word[i] = swab(key_tmp[i]);
2110
2111         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2112         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2113                                 CRYPTO_TFM_REQ_MASK);
2114         ret = crypto_cipher_setkey(ctx->kaes,
2115                                    (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2116                                    AES_MIN_KEY_SIZE);
2117         if (ret)
2118                 return ret;
2119
2120         ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2121         ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2122         ctx->cbcmac = false;
2123
2124         memzero_explicit(&aes, sizeof(aes));
2125         return 0;
2126 }
2127
2128 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2129 {
2130         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2131
2132         safexcel_ahash_cra_init(tfm);
2133         ctx->kaes = crypto_alloc_cipher("aes", 0, 0);
2134         return PTR_ERR_OR_ZERO(ctx->kaes);
2135 }
2136
2137 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2138 {
2139         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2140
2141         crypto_free_cipher(ctx->kaes);
2142         safexcel_ahash_cra_exit(tfm);
2143 }
2144
2145 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2146         .type = SAFEXCEL_ALG_TYPE_AHASH,
2147         .algo_mask = 0,
2148         .alg.ahash = {
2149                 .init = safexcel_cbcmac_init,
2150                 .update = safexcel_ahash_update,
2151                 .final = safexcel_ahash_final,
2152                 .finup = safexcel_ahash_finup,
2153                 .digest = safexcel_cbcmac_digest,
2154                 .setkey = safexcel_xcbcmac_setkey,
2155                 .export = safexcel_ahash_export,
2156                 .import = safexcel_ahash_import,
2157                 .halg = {
2158                         .digestsize = AES_BLOCK_SIZE,
2159                         .statesize = sizeof(struct safexcel_ahash_export_state),
2160                         .base = {
2161                                 .cra_name = "xcbc(aes)",
2162                                 .cra_driver_name = "safexcel-xcbc-aes",
2163                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2164                                 .cra_flags = CRYPTO_ALG_ASYNC |
2165                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2166                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2167                                 .cra_blocksize = AES_BLOCK_SIZE,
2168                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2169                                 .cra_init = safexcel_xcbcmac_cra_init,
2170                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2171                                 .cra_module = THIS_MODULE,
2172                         },
2173                 },
2174         },
2175 };
2176
2177 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2178                                 unsigned int len)
2179 {
2180         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2181         struct crypto_aes_ctx aes;
2182         __be64 consts[4];
2183         u64 _const[2];
2184         u8 msb_mask, gfmask;
2185         int ret, i;
2186
2187         ret = aes_expandkey(&aes, key, len);
2188         if (ret)
2189                 return ret;
2190
2191         for (i = 0; i < len / sizeof(u32); i++)
2192                 ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]);
2193
2194         /* precompute the CMAC key material */
2195         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2196         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2197                                 CRYPTO_TFM_REQ_MASK);
2198         ret = crypto_cipher_setkey(ctx->kaes, key, len);
2199         if (ret)
2200                 return ret;
2201
2202         /* code below borrowed from crypto/cmac.c */
2203         /* encrypt the zero block */
2204         memset(consts, 0, AES_BLOCK_SIZE);
2205         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts);
2206
2207         gfmask = 0x87;
2208         _const[0] = be64_to_cpu(consts[1]);
2209         _const[1] = be64_to_cpu(consts[0]);
2210
2211         /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2212         for (i = 0; i < 4; i += 2) {
2213                 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2214                 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2215                 _const[0] = (_const[0] << 1) ^ msb_mask;
2216
2217                 consts[i + 0] = cpu_to_be64(_const[1]);
2218                 consts[i + 1] = cpu_to_be64(_const[0]);
2219         }
2220         /* end of code borrowed from crypto/cmac.c */
2221
2222         for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2223                 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2224
2225         if (len == AES_KEYSIZE_192) {
2226                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2227                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2228         } else if (len == AES_KEYSIZE_256) {
2229                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2230                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2231         } else {
2232                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2233                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2234         }
2235         ctx->cbcmac = false;
2236
2237         memzero_explicit(&aes, sizeof(aes));
2238         return 0;
2239 }
2240
2241 struct safexcel_alg_template safexcel_alg_cmac = {
2242         .type = SAFEXCEL_ALG_TYPE_AHASH,
2243         .algo_mask = 0,
2244         .alg.ahash = {
2245                 .init = safexcel_cbcmac_init,
2246                 .update = safexcel_ahash_update,
2247                 .final = safexcel_ahash_final,
2248                 .finup = safexcel_ahash_finup,
2249                 .digest = safexcel_cbcmac_digest,
2250                 .setkey = safexcel_cmac_setkey,
2251                 .export = safexcel_ahash_export,
2252                 .import = safexcel_ahash_import,
2253                 .halg = {
2254                         .digestsize = AES_BLOCK_SIZE,
2255                         .statesize = sizeof(struct safexcel_ahash_export_state),
2256                         .base = {
2257                                 .cra_name = "cmac(aes)",
2258                                 .cra_driver_name = "safexcel-cmac-aes",
2259                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2260                                 .cra_flags = CRYPTO_ALG_ASYNC |
2261                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2262                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2263                                 .cra_blocksize = AES_BLOCK_SIZE,
2264                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2265                                 .cra_init = safexcel_xcbcmac_cra_init,
2266                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2267                                 .cra_module = THIS_MODULE,
2268                         },
2269                 },
2270         },
2271 };
2272
2273 static int safexcel_sm3_init(struct ahash_request *areq)
2274 {
2275         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2276         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2277
2278         memset(req, 0, sizeof(*req));
2279
2280         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2281         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2282         req->state_sz = SM3_DIGEST_SIZE;
2283         req->digest_sz = SM3_DIGEST_SIZE;
2284         req->block_sz = SM3_BLOCK_SIZE;
2285
2286         return 0;
2287 }
2288
2289 static int safexcel_sm3_digest(struct ahash_request *areq)
2290 {
2291         int ret = safexcel_sm3_init(areq);
2292
2293         if (ret)
2294                 return ret;
2295
2296         return safexcel_ahash_finup(areq);
2297 }
2298
2299 struct safexcel_alg_template safexcel_alg_sm3 = {
2300         .type = SAFEXCEL_ALG_TYPE_AHASH,
2301         .algo_mask = SAFEXCEL_ALG_SM3,
2302         .alg.ahash = {
2303                 .init = safexcel_sm3_init,
2304                 .update = safexcel_ahash_update,
2305                 .final = safexcel_ahash_final,
2306                 .finup = safexcel_ahash_finup,
2307                 .digest = safexcel_sm3_digest,
2308                 .export = safexcel_ahash_export,
2309                 .import = safexcel_ahash_import,
2310                 .halg = {
2311                         .digestsize = SM3_DIGEST_SIZE,
2312                         .statesize = sizeof(struct safexcel_ahash_export_state),
2313                         .base = {
2314                                 .cra_name = "sm3",
2315                                 .cra_driver_name = "safexcel-sm3",
2316                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2317                                 .cra_flags = CRYPTO_ALG_ASYNC |
2318                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2319                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2320                                 .cra_blocksize = SM3_BLOCK_SIZE,
2321                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2322                                 .cra_init = safexcel_ahash_cra_init,
2323                                 .cra_exit = safexcel_ahash_cra_exit,
2324                                 .cra_module = THIS_MODULE,
2325                         },
2326                 },
2327         },
2328 };
2329
2330 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2331                                     unsigned int keylen)
2332 {
2333         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2334                                         SM3_DIGEST_SIZE);
2335 }
2336
2337 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2338 {
2339         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2340         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2341
2342         memset(req, 0, sizeof(*req));
2343
2344         /* Start from ipad precompute */
2345         memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2346         /* Already processed the key^ipad part now! */
2347         req->len        = SM3_BLOCK_SIZE;
2348         req->processed  = SM3_BLOCK_SIZE;
2349
2350         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2351         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2352         req->state_sz = SM3_DIGEST_SIZE;
2353         req->digest_sz = SM3_DIGEST_SIZE;
2354         req->block_sz = SM3_BLOCK_SIZE;
2355         req->hmac = true;
2356
2357         return 0;
2358 }
2359
2360 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2361 {
2362         int ret = safexcel_hmac_sm3_init(areq);
2363
2364         if (ret)
2365                 return ret;
2366
2367         return safexcel_ahash_finup(areq);
2368 }
2369
2370 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2371         .type = SAFEXCEL_ALG_TYPE_AHASH,
2372         .algo_mask = SAFEXCEL_ALG_SM3,
2373         .alg.ahash = {
2374                 .init = safexcel_hmac_sm3_init,
2375                 .update = safexcel_ahash_update,
2376                 .final = safexcel_ahash_final,
2377                 .finup = safexcel_ahash_finup,
2378                 .digest = safexcel_hmac_sm3_digest,
2379                 .setkey = safexcel_hmac_sm3_setkey,
2380                 .export = safexcel_ahash_export,
2381                 .import = safexcel_ahash_import,
2382                 .halg = {
2383                         .digestsize = SM3_DIGEST_SIZE,
2384                         .statesize = sizeof(struct safexcel_ahash_export_state),
2385                         .base = {
2386                                 .cra_name = "hmac(sm3)",
2387                                 .cra_driver_name = "safexcel-hmac-sm3",
2388                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2389                                 .cra_flags = CRYPTO_ALG_ASYNC |
2390                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2391                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2392                                 .cra_blocksize = SM3_BLOCK_SIZE,
2393                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2394                                 .cra_init = safexcel_ahash_cra_init,
2395                                 .cra_exit = safexcel_ahash_cra_exit,
2396                                 .cra_module = THIS_MODULE,
2397                         },
2398                 },
2399         },
2400 };
2401
2402 static int safexcel_sha3_224_init(struct ahash_request *areq)
2403 {
2404         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2405         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2406         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2407
2408         memset(req, 0, sizeof(*req));
2409
2410         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2411         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2412         req->state_sz = SHA3_224_DIGEST_SIZE;
2413         req->digest_sz = SHA3_224_DIGEST_SIZE;
2414         req->block_sz = SHA3_224_BLOCK_SIZE;
2415         ctx->do_fallback = false;
2416         ctx->fb_init_done = false;
2417         return 0;
2418 }
2419
2420 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2421 {
2422         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2423         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2424         struct ahash_request *subreq = ahash_request_ctx(req);
2425         int ret = 0;
2426
2427         if (ctx->do_fallback) {
2428                 ahash_request_set_tfm(subreq, ctx->fback);
2429                 ahash_request_set_callback(subreq, req->base.flags,
2430                                            req->base.complete, req->base.data);
2431                 ahash_request_set_crypt(subreq, req->src, req->result,
2432                                         req->nbytes);
2433                 if (!ctx->fb_init_done) {
2434                         if (ctx->fb_do_setkey) {
2435                                 /* Set fallback cipher HMAC key */
2436                                 u8 key[SHA3_224_BLOCK_SIZE];
2437
2438                                 memcpy(key, &ctx->base.ipad,
2439                                        crypto_ahash_blocksize(ctx->fback) / 2);
2440                                 memcpy(key +
2441                                        crypto_ahash_blocksize(ctx->fback) / 2,
2442                                        &ctx->base.opad,
2443                                        crypto_ahash_blocksize(ctx->fback) / 2);
2444                                 ret = crypto_ahash_setkey(ctx->fback, key,
2445                                         crypto_ahash_blocksize(ctx->fback));
2446                                 memzero_explicit(key,
2447                                         crypto_ahash_blocksize(ctx->fback));
2448                                 ctx->fb_do_setkey = false;
2449                         }
2450                         ret = ret ?: crypto_ahash_init(subreq);
2451                         ctx->fb_init_done = true;
2452                 }
2453         }
2454         return ret;
2455 }
2456
2457 static int safexcel_sha3_update(struct ahash_request *req)
2458 {
2459         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2460         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2461         struct ahash_request *subreq = ahash_request_ctx(req);
2462
2463         ctx->do_fallback = true;
2464         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2465 }
2466
2467 static int safexcel_sha3_final(struct ahash_request *req)
2468 {
2469         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2470         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2471         struct ahash_request *subreq = ahash_request_ctx(req);
2472
2473         ctx->do_fallback = true;
2474         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2475 }
2476
2477 static int safexcel_sha3_finup(struct ahash_request *req)
2478 {
2479         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2480         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2481         struct ahash_request *subreq = ahash_request_ctx(req);
2482
2483         ctx->do_fallback |= !req->nbytes;
2484         if (ctx->do_fallback)
2485                 /* Update or ex/import happened or len 0, cannot use the HW */
2486                 return safexcel_sha3_fbcheck(req) ?:
2487                        crypto_ahash_finup(subreq);
2488         else
2489                 return safexcel_ahash_finup(req);
2490 }
2491
2492 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2493 {
2494         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2495         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2496         struct ahash_request *subreq = ahash_request_ctx(req);
2497
2498         ctx->do_fallback = true;
2499         ctx->fb_init_done = false;
2500         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2501 }
2502
2503 static int safexcel_sha3_224_digest(struct ahash_request *req)
2504 {
2505         if (req->nbytes)
2506                 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2507
2508         /* HW cannot do zero length hash, use fallback instead */
2509         return safexcel_sha3_digest_fallback(req);
2510 }
2511
2512 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2513 {
2514         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2515         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2516         struct ahash_request *subreq = ahash_request_ctx(req);
2517
2518         ctx->do_fallback = true;
2519         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2520 }
2521
2522 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2523 {
2524         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2525         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2526         struct ahash_request *subreq = ahash_request_ctx(req);
2527
2528         ctx->do_fallback = true;
2529         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2530         // return safexcel_ahash_import(req, in);
2531 }
2532
2533 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2534 {
2535         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2536         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2537
2538         safexcel_ahash_cra_init(tfm);
2539
2540         /* Allocate fallback implementation */
2541         ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2542                                         CRYPTO_ALG_ASYNC |
2543                                         CRYPTO_ALG_NEED_FALLBACK);
2544         if (IS_ERR(ctx->fback))
2545                 return PTR_ERR(ctx->fback);
2546
2547         /* Update statesize from fallback algorithm! */
2548         crypto_hash_alg_common(ahash)->statesize =
2549                 crypto_ahash_statesize(ctx->fback);
2550         crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
2551                                             sizeof(struct ahash_request) +
2552                                             crypto_ahash_reqsize(ctx->fback)));
2553         return 0;
2554 }
2555
2556 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2557 {
2558         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2559
2560         crypto_free_ahash(ctx->fback);
2561         safexcel_ahash_cra_exit(tfm);
2562 }
2563
2564 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2565         .type = SAFEXCEL_ALG_TYPE_AHASH,
2566         .algo_mask = SAFEXCEL_ALG_SHA3,
2567         .alg.ahash = {
2568                 .init = safexcel_sha3_224_init,
2569                 .update = safexcel_sha3_update,
2570                 .final = safexcel_sha3_final,
2571                 .finup = safexcel_sha3_finup,
2572                 .digest = safexcel_sha3_224_digest,
2573                 .export = safexcel_sha3_export,
2574                 .import = safexcel_sha3_import,
2575                 .halg = {
2576                         .digestsize = SHA3_224_DIGEST_SIZE,
2577                         .statesize = sizeof(struct safexcel_ahash_export_state),
2578                         .base = {
2579                                 .cra_name = "sha3-224",
2580                                 .cra_driver_name = "safexcel-sha3-224",
2581                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2582                                 .cra_flags = CRYPTO_ALG_ASYNC |
2583                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2584                                              CRYPTO_ALG_NEED_FALLBACK,
2585                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2586                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2587                                 .cra_init = safexcel_sha3_cra_init,
2588                                 .cra_exit = safexcel_sha3_cra_exit,
2589                                 .cra_module = THIS_MODULE,
2590                         },
2591                 },
2592         },
2593 };
2594
2595 static int safexcel_sha3_256_init(struct ahash_request *areq)
2596 {
2597         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2598         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2599         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2600
2601         memset(req, 0, sizeof(*req));
2602
2603         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2604         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2605         req->state_sz = SHA3_256_DIGEST_SIZE;
2606         req->digest_sz = SHA3_256_DIGEST_SIZE;
2607         req->block_sz = SHA3_256_BLOCK_SIZE;
2608         ctx->do_fallback = false;
2609         ctx->fb_init_done = false;
2610         return 0;
2611 }
2612
2613 static int safexcel_sha3_256_digest(struct ahash_request *req)
2614 {
2615         if (req->nbytes)
2616                 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2617
2618         /* HW cannot do zero length hash, use fallback instead */
2619         return safexcel_sha3_digest_fallback(req);
2620 }
2621
2622 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2623         .type = SAFEXCEL_ALG_TYPE_AHASH,
2624         .algo_mask = SAFEXCEL_ALG_SHA3,
2625         .alg.ahash = {
2626                 .init = safexcel_sha3_256_init,
2627                 .update = safexcel_sha3_update,
2628                 .final = safexcel_sha3_final,
2629                 .finup = safexcel_sha3_finup,
2630                 .digest = safexcel_sha3_256_digest,
2631                 .export = safexcel_sha3_export,
2632                 .import = safexcel_sha3_import,
2633                 .halg = {
2634                         .digestsize = SHA3_256_DIGEST_SIZE,
2635                         .statesize = sizeof(struct safexcel_ahash_export_state),
2636                         .base = {
2637                                 .cra_name = "sha3-256",
2638                                 .cra_driver_name = "safexcel-sha3-256",
2639                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2640                                 .cra_flags = CRYPTO_ALG_ASYNC |
2641                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2642                                              CRYPTO_ALG_NEED_FALLBACK,
2643                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2644                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2645                                 .cra_init = safexcel_sha3_cra_init,
2646                                 .cra_exit = safexcel_sha3_cra_exit,
2647                                 .cra_module = THIS_MODULE,
2648                         },
2649                 },
2650         },
2651 };
2652
2653 static int safexcel_sha3_384_init(struct ahash_request *areq)
2654 {
2655         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2656         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2657         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2658
2659         memset(req, 0, sizeof(*req));
2660
2661         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2662         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2663         req->state_sz = SHA3_384_DIGEST_SIZE;
2664         req->digest_sz = SHA3_384_DIGEST_SIZE;
2665         req->block_sz = SHA3_384_BLOCK_SIZE;
2666         ctx->do_fallback = false;
2667         ctx->fb_init_done = false;
2668         return 0;
2669 }
2670
2671 static int safexcel_sha3_384_digest(struct ahash_request *req)
2672 {
2673         if (req->nbytes)
2674                 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2675
2676         /* HW cannot do zero length hash, use fallback instead */
2677         return safexcel_sha3_digest_fallback(req);
2678 }
2679
2680 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2681         .type = SAFEXCEL_ALG_TYPE_AHASH,
2682         .algo_mask = SAFEXCEL_ALG_SHA3,
2683         .alg.ahash = {
2684                 .init = safexcel_sha3_384_init,
2685                 .update = safexcel_sha3_update,
2686                 .final = safexcel_sha3_final,
2687                 .finup = safexcel_sha3_finup,
2688                 .digest = safexcel_sha3_384_digest,
2689                 .export = safexcel_sha3_export,
2690                 .import = safexcel_sha3_import,
2691                 .halg = {
2692                         .digestsize = SHA3_384_DIGEST_SIZE,
2693                         .statesize = sizeof(struct safexcel_ahash_export_state),
2694                         .base = {
2695                                 .cra_name = "sha3-384",
2696                                 .cra_driver_name = "safexcel-sha3-384",
2697                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2698                                 .cra_flags = CRYPTO_ALG_ASYNC |
2699                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2700                                              CRYPTO_ALG_NEED_FALLBACK,
2701                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2702                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2703                                 .cra_init = safexcel_sha3_cra_init,
2704                                 .cra_exit = safexcel_sha3_cra_exit,
2705                                 .cra_module = THIS_MODULE,
2706                         },
2707                 },
2708         },
2709 };
2710
2711 static int safexcel_sha3_512_init(struct ahash_request *areq)
2712 {
2713         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2714         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2715         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2716
2717         memset(req, 0, sizeof(*req));
2718
2719         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2720         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2721         req->state_sz = SHA3_512_DIGEST_SIZE;
2722         req->digest_sz = SHA3_512_DIGEST_SIZE;
2723         req->block_sz = SHA3_512_BLOCK_SIZE;
2724         ctx->do_fallback = false;
2725         ctx->fb_init_done = false;
2726         return 0;
2727 }
2728
2729 static int safexcel_sha3_512_digest(struct ahash_request *req)
2730 {
2731         if (req->nbytes)
2732                 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2733
2734         /* HW cannot do zero length hash, use fallback instead */
2735         return safexcel_sha3_digest_fallback(req);
2736 }
2737
2738 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2739         .type = SAFEXCEL_ALG_TYPE_AHASH,
2740         .algo_mask = SAFEXCEL_ALG_SHA3,
2741         .alg.ahash = {
2742                 .init = safexcel_sha3_512_init,
2743                 .update = safexcel_sha3_update,
2744                 .final = safexcel_sha3_final,
2745                 .finup = safexcel_sha3_finup,
2746                 .digest = safexcel_sha3_512_digest,
2747                 .export = safexcel_sha3_export,
2748                 .import = safexcel_sha3_import,
2749                 .halg = {
2750                         .digestsize = SHA3_512_DIGEST_SIZE,
2751                         .statesize = sizeof(struct safexcel_ahash_export_state),
2752                         .base = {
2753                                 .cra_name = "sha3-512",
2754                                 .cra_driver_name = "safexcel-sha3-512",
2755                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2756                                 .cra_flags = CRYPTO_ALG_ASYNC |
2757                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2758                                              CRYPTO_ALG_NEED_FALLBACK,
2759                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2760                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2761                                 .cra_init = safexcel_sha3_cra_init,
2762                                 .cra_exit = safexcel_sha3_cra_exit,
2763                                 .cra_module = THIS_MODULE,
2764                         },
2765                 },
2766         },
2767 };
2768
2769 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2770 {
2771         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2772         int ret;
2773
2774         ret = safexcel_sha3_cra_init(tfm);
2775         if (ret)
2776                 return ret;
2777
2778         /* Allocate precalc basic digest implementation */
2779         ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2780         if (IS_ERR(ctx->shpre))
2781                 return PTR_ERR(ctx->shpre);
2782
2783         ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2784                               crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2785         if (!ctx->shdesc) {
2786                 crypto_free_shash(ctx->shpre);
2787                 return -ENOMEM;
2788         }
2789         ctx->shdesc->tfm = ctx->shpre;
2790         return 0;
2791 }
2792
2793 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2794 {
2795         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2796
2797         crypto_free_ahash(ctx->fback);
2798         crypto_free_shash(ctx->shpre);
2799         kfree(ctx->shdesc);
2800         safexcel_ahash_cra_exit(tfm);
2801 }
2802
2803 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2804                                      unsigned int keylen)
2805 {
2806         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2807         int ret = 0;
2808
2809         if (keylen > crypto_ahash_blocksize(tfm)) {
2810                 /*
2811                  * If the key is larger than the blocksize, then hash it
2812                  * first using our fallback cipher
2813                  */
2814                 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2815                                           ctx->base.ipad.byte);
2816                 keylen = crypto_shash_digestsize(ctx->shpre);
2817
2818                 /*
2819                  * If the digest is larger than half the blocksize, we need to
2820                  * move the rest to opad due to the way our HMAC infra works.
2821                  */
2822                 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2823                         /* Buffers overlap, need to use memmove iso memcpy! */
2824                         memmove(&ctx->base.opad,
2825                                 ctx->base.ipad.byte +
2826                                         crypto_ahash_blocksize(tfm) / 2,
2827                                 keylen - crypto_ahash_blocksize(tfm) / 2);
2828         } else {
2829                 /*
2830                  * Copy the key to our ipad & opad buffers
2831                  * Note that ipad and opad each contain one half of the key,
2832                  * to match the existing HMAC driver infrastructure.
2833                  */
2834                 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2835                         memcpy(&ctx->base.ipad, key, keylen);
2836                 } else {
2837                         memcpy(&ctx->base.ipad, key,
2838                                crypto_ahash_blocksize(tfm) / 2);
2839                         memcpy(&ctx->base.opad,
2840                                key + crypto_ahash_blocksize(tfm) / 2,
2841                                keylen - crypto_ahash_blocksize(tfm) / 2);
2842                 }
2843         }
2844
2845         /* Pad key with zeroes */
2846         if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2847                 memset(ctx->base.ipad.byte + keylen, 0,
2848                        crypto_ahash_blocksize(tfm) / 2 - keylen);
2849                 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2850         } else {
2851                 memset(ctx->base.opad.byte + keylen -
2852                        crypto_ahash_blocksize(tfm) / 2, 0,
2853                        crypto_ahash_blocksize(tfm) - keylen);
2854         }
2855
2856         /* If doing fallback, still need to set the new key! */
2857         ctx->fb_do_setkey = true;
2858         return ret;
2859 }
2860
2861 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2862 {
2863         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2864         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2865         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2866
2867         memset(req, 0, sizeof(*req));
2868
2869         /* Copy (half of) the key */
2870         memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2871         /* Start of HMAC should have len == processed == blocksize */
2872         req->len        = SHA3_224_BLOCK_SIZE;
2873         req->processed  = SHA3_224_BLOCK_SIZE;
2874         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2875         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2876         req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2877         req->digest_sz = SHA3_224_DIGEST_SIZE;
2878         req->block_sz = SHA3_224_BLOCK_SIZE;
2879         req->hmac = true;
2880         ctx->do_fallback = false;
2881         ctx->fb_init_done = false;
2882         return 0;
2883 }
2884
2885 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2886 {
2887         if (req->nbytes)
2888                 return safexcel_hmac_sha3_224_init(req) ?:
2889                        safexcel_ahash_finup(req);
2890
2891         /* HW cannot do zero length HMAC, use fallback instead */
2892         return safexcel_sha3_digest_fallback(req);
2893 }
2894
2895 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2896 {
2897         return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2898 }
2899
2900 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2901         .type = SAFEXCEL_ALG_TYPE_AHASH,
2902         .algo_mask = SAFEXCEL_ALG_SHA3,
2903         .alg.ahash = {
2904                 .init = safexcel_hmac_sha3_224_init,
2905                 .update = safexcel_sha3_update,
2906                 .final = safexcel_sha3_final,
2907                 .finup = safexcel_sha3_finup,
2908                 .digest = safexcel_hmac_sha3_224_digest,
2909                 .setkey = safexcel_hmac_sha3_setkey,
2910                 .export = safexcel_sha3_export,
2911                 .import = safexcel_sha3_import,
2912                 .halg = {
2913                         .digestsize = SHA3_224_DIGEST_SIZE,
2914                         .statesize = sizeof(struct safexcel_ahash_export_state),
2915                         .base = {
2916                                 .cra_name = "hmac(sha3-224)",
2917                                 .cra_driver_name = "safexcel-hmac-sha3-224",
2918                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2919                                 .cra_flags = CRYPTO_ALG_ASYNC |
2920                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2921                                              CRYPTO_ALG_NEED_FALLBACK,
2922                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2923                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2924                                 .cra_init = safexcel_hmac_sha3_224_cra_init,
2925                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2926                                 .cra_module = THIS_MODULE,
2927                         },
2928                 },
2929         },
2930 };
2931
2932 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2933 {
2934         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2935         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2936         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2937
2938         memset(req, 0, sizeof(*req));
2939
2940         /* Copy (half of) the key */
2941         memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2942         /* Start of HMAC should have len == processed == blocksize */
2943         req->len        = SHA3_256_BLOCK_SIZE;
2944         req->processed  = SHA3_256_BLOCK_SIZE;
2945         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2946         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2947         req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2948         req->digest_sz = SHA3_256_DIGEST_SIZE;
2949         req->block_sz = SHA3_256_BLOCK_SIZE;
2950         req->hmac = true;
2951         ctx->do_fallback = false;
2952         ctx->fb_init_done = false;
2953         return 0;
2954 }
2955
2956 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2957 {
2958         if (req->nbytes)
2959                 return safexcel_hmac_sha3_256_init(req) ?:
2960                        safexcel_ahash_finup(req);
2961
2962         /* HW cannot do zero length HMAC, use fallback instead */
2963         return safexcel_sha3_digest_fallback(req);
2964 }
2965
2966 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2967 {
2968         return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2969 }
2970
2971 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2972         .type = SAFEXCEL_ALG_TYPE_AHASH,
2973         .algo_mask = SAFEXCEL_ALG_SHA3,
2974         .alg.ahash = {
2975                 .init = safexcel_hmac_sha3_256_init,
2976                 .update = safexcel_sha3_update,
2977                 .final = safexcel_sha3_final,
2978                 .finup = safexcel_sha3_finup,
2979                 .digest = safexcel_hmac_sha3_256_digest,
2980                 .setkey = safexcel_hmac_sha3_setkey,
2981                 .export = safexcel_sha3_export,
2982                 .import = safexcel_sha3_import,
2983                 .halg = {
2984                         .digestsize = SHA3_256_DIGEST_SIZE,
2985                         .statesize = sizeof(struct safexcel_ahash_export_state),
2986                         .base = {
2987                                 .cra_name = "hmac(sha3-256)",
2988                                 .cra_driver_name = "safexcel-hmac-sha3-256",
2989                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2990                                 .cra_flags = CRYPTO_ALG_ASYNC |
2991                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2992                                              CRYPTO_ALG_NEED_FALLBACK,
2993                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2994                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2995                                 .cra_init = safexcel_hmac_sha3_256_cra_init,
2996                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2997                                 .cra_module = THIS_MODULE,
2998                         },
2999                 },
3000         },
3001 };
3002
3003 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
3004 {
3005         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3006         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3007         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3008
3009         memset(req, 0, sizeof(*req));
3010
3011         /* Copy (half of) the key */
3012         memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
3013         /* Start of HMAC should have len == processed == blocksize */
3014         req->len        = SHA3_384_BLOCK_SIZE;
3015         req->processed  = SHA3_384_BLOCK_SIZE;
3016         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
3017         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3018         req->state_sz = SHA3_384_BLOCK_SIZE / 2;
3019         req->digest_sz = SHA3_384_DIGEST_SIZE;
3020         req->block_sz = SHA3_384_BLOCK_SIZE;
3021         req->hmac = true;
3022         ctx->do_fallback = false;
3023         ctx->fb_init_done = false;
3024         return 0;
3025 }
3026
3027 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
3028 {
3029         if (req->nbytes)
3030                 return safexcel_hmac_sha3_384_init(req) ?:
3031                        safexcel_ahash_finup(req);
3032
3033         /* HW cannot do zero length HMAC, use fallback instead */
3034         return safexcel_sha3_digest_fallback(req);
3035 }
3036
3037 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3038 {
3039         return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3040 }
3041
3042 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3043         .type = SAFEXCEL_ALG_TYPE_AHASH,
3044         .algo_mask = SAFEXCEL_ALG_SHA3,
3045         .alg.ahash = {
3046                 .init = safexcel_hmac_sha3_384_init,
3047                 .update = safexcel_sha3_update,
3048                 .final = safexcel_sha3_final,
3049                 .finup = safexcel_sha3_finup,
3050                 .digest = safexcel_hmac_sha3_384_digest,
3051                 .setkey = safexcel_hmac_sha3_setkey,
3052                 .export = safexcel_sha3_export,
3053                 .import = safexcel_sha3_import,
3054                 .halg = {
3055                         .digestsize = SHA3_384_DIGEST_SIZE,
3056                         .statesize = sizeof(struct safexcel_ahash_export_state),
3057                         .base = {
3058                                 .cra_name = "hmac(sha3-384)",
3059                                 .cra_driver_name = "safexcel-hmac-sha3-384",
3060                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3061                                 .cra_flags = CRYPTO_ALG_ASYNC |
3062                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3063                                              CRYPTO_ALG_NEED_FALLBACK,
3064                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3065                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3066                                 .cra_init = safexcel_hmac_sha3_384_cra_init,
3067                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3068                                 .cra_module = THIS_MODULE,
3069                         },
3070                 },
3071         },
3072 };
3073
3074 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3075 {
3076         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3077         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3078         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3079
3080         memset(req, 0, sizeof(*req));
3081
3082         /* Copy (half of) the key */
3083         memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3084         /* Start of HMAC should have len == processed == blocksize */
3085         req->len        = SHA3_512_BLOCK_SIZE;
3086         req->processed  = SHA3_512_BLOCK_SIZE;
3087         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3088         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3089         req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3090         req->digest_sz = SHA3_512_DIGEST_SIZE;
3091         req->block_sz = SHA3_512_BLOCK_SIZE;
3092         req->hmac = true;
3093         ctx->do_fallback = false;
3094         ctx->fb_init_done = false;
3095         return 0;
3096 }
3097
3098 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3099 {
3100         if (req->nbytes)
3101                 return safexcel_hmac_sha3_512_init(req) ?:
3102                        safexcel_ahash_finup(req);
3103
3104         /* HW cannot do zero length HMAC, use fallback instead */
3105         return safexcel_sha3_digest_fallback(req);
3106 }
3107
3108 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3109 {
3110         return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3111 }
3112 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3113         .type = SAFEXCEL_ALG_TYPE_AHASH,
3114         .algo_mask = SAFEXCEL_ALG_SHA3,
3115         .alg.ahash = {
3116                 .init = safexcel_hmac_sha3_512_init,
3117                 .update = safexcel_sha3_update,
3118                 .final = safexcel_sha3_final,
3119                 .finup = safexcel_sha3_finup,
3120                 .digest = safexcel_hmac_sha3_512_digest,
3121                 .setkey = safexcel_hmac_sha3_setkey,
3122                 .export = safexcel_sha3_export,
3123                 .import = safexcel_sha3_import,
3124                 .halg = {
3125                         .digestsize = SHA3_512_DIGEST_SIZE,
3126                         .statesize = sizeof(struct safexcel_ahash_export_state),
3127                         .base = {
3128                                 .cra_name = "hmac(sha3-512)",
3129                                 .cra_driver_name = "safexcel-hmac-sha3-512",
3130                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3131                                 .cra_flags = CRYPTO_ALG_ASYNC |
3132                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3133                                              CRYPTO_ALG_NEED_FALLBACK,
3134                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3135                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3136                                 .cra_init = safexcel_hmac_sha3_512_cra_init,
3137                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3138                                 .cra_module = THIS_MODULE,
3139                         },
3140                 },
3141         },
3142 };