drivers/crypto/starfive/jh7110-hash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Hash function and HMAC support for StarFive driver
   4  *
   5  * Copyright (c) 2022 StarFive Technology
   6  *
   7  */
   8
   9 #include <crypto/engine.h>
  10 #include <crypto/internal/hash.h>
  11 #include <crypto/scatterwalk.h>
  12 #include "jh7110-cryp.h"
  13 #include <linux/amba/pl080.h>
  14 #include <linux/clk.h>
  15 #include <linux/dma-direct.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/iopoll.h>
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/pm_runtime.h>
  22 #include <linux/reset.h>
  23
  24 #define STARFIVE_HASH_REGS_OFFSET       0x300
  25 #define STARFIVE_HASH_SHACSR            (STARFIVE_HASH_REGS_OFFSET + 0x0)
  26 #define STARFIVE_HASH_SHAWDR            (STARFIVE_HASH_REGS_OFFSET + 0x4)
  27 #define STARFIVE_HASH_SHARDR            (STARFIVE_HASH_REGS_OFFSET + 0x8)
  28 #define STARFIVE_HASH_SHAWSR            (STARFIVE_HASH_REGS_OFFSET + 0xC)
  29 #define STARFIVE_HASH_SHAWLEN3          (STARFIVE_HASH_REGS_OFFSET + 0x10)
  30 #define STARFIVE_HASH_SHAWLEN2          (STARFIVE_HASH_REGS_OFFSET + 0x14)
  31 #define STARFIVE_HASH_SHAWLEN1          (STARFIVE_HASH_REGS_OFFSET + 0x18)
  32 #define STARFIVE_HASH_SHAWLEN0          (STARFIVE_HASH_REGS_OFFSET + 0x1C)
  33 #define STARFIVE_HASH_SHAWKR            (STARFIVE_HASH_REGS_OFFSET + 0x20)
  34 #define STARFIVE_HASH_SHAWKLEN          (STARFIVE_HASH_REGS_OFFSET + 0x24)
  35
  36 #define STARFIVE_HASH_BUFLEN            SHA512_BLOCK_SIZE
  37 #define STARFIVE_HASH_RESET             0x2
  38
  39 static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx)
  40 {
  41         struct starfive_cryp_dev *cryp = ctx->cryp;
  42         u32 status;
  43
  44         return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
  45                                           !(status & STARFIVE_HASH_BUSY), 10, 100000);
  46 }
  47
  48 static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx)
  49 {
  50         struct starfive_cryp_dev *cryp = ctx->cryp;
  51         u32 status;
  52
  53         return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
  54                                           (status & STARFIVE_HASH_KEY_DONE), 10, 100000);
  55 }
  56
  57 static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx)
  58 {
  59         struct starfive_cryp_request_ctx *rctx = ctx->rctx;
  60         struct starfive_cryp_dev *cryp = ctx->cryp;
  61         int klen = ctx->keylen, loop;
  62         unsigned int *key = (unsigned int *)ctx->key;
  63         unsigned char *cl;
  64
  65         writel(ctx->keylen, cryp->base + STARFIVE_HASH_SHAWKLEN);
  66
  67         rctx->csr.hash.hmac = 1;
  68         rctx->csr.hash.key_flag = 1;
  69
  70         writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
  71
  72         for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++)
  73                 writel(*key, cryp->base + STARFIVE_HASH_SHAWKR);
  74
  75         if (klen & 0x3) {
  76                 cl = (unsigned char *)key;
  77                 for (loop = 0; loop < (klen & 0x3); loop++, cl++)
  78                         writeb(*cl, cryp->base + STARFIVE_HASH_SHAWKR);
  79         }
  80
  81         if (starfive_hash_wait_key_done(ctx))
  82                 return dev_err_probe(cryp->dev, -ETIMEDOUT, "starfive_hash_wait_key_done error\n");
  83
  84         return 0;
  85 }
  86
  87 static void starfive_hash_start(void *param)
  88 {
  89         struct starfive_cryp_ctx *ctx = param;
  90         struct starfive_cryp_request_ctx *rctx = ctx->rctx;
  91         struct starfive_cryp_dev *cryp = ctx->cryp;
  92         union starfive_alg_cr alg_cr;
  93         union starfive_hash_csr csr;
  94         u32 stat;
  95
  96         dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
  97
  98         alg_cr.v = 0;
  99         alg_cr.clear = 1;
 100
 101         writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
 102
 103         csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR);
 104         csr.firstb = 0;
 105         csr.final = 1;
 106
 107         stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
 108         stat &= ~STARFIVE_IE_MASK_HASH_DONE;
 109         writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
 110         writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR);
 111 }
 112
 113 static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx)
 114 {
 115         struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 116         struct starfive_cryp_dev *cryp = ctx->cryp;
 117         struct dma_async_tx_descriptor  *in_desc;
 118         union  starfive_alg_cr alg_cr;
 119         int total_len;
 120         int ret;
 121
 122         if (!rctx->total) {
 123                 starfive_hash_start(ctx);
 124                 return 0;
 125         }
 126
 127         writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
 128
 129         total_len = rctx->total;
 130         total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
 131         sg_dma_len(rctx->in_sg) = total_len;
 132
 133         alg_cr.v = 0;
 134         alg_cr.start = 1;
 135         alg_cr.hash_dma_en = 1;
 136
 137         writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
 138
 139         ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
 140         if (!ret)
 141                 return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n");
 142
 143         cryp->cfg_in.direction = DMA_MEM_TO_DEV;
 144         cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 145         cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 146         cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
 147         cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
 148         cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
 149
 150         dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
 151
 152         in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg,
 153                                           ret, DMA_MEM_TO_DEV,
 154                                           DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
 155
 156         if (!in_desc)
 157                 return -EINVAL;
 158
 159         in_desc->callback = starfive_hash_start;
 160         in_desc->callback_param = ctx;
 161
 162         dmaengine_submit(in_desc);
 163         dma_async_issue_pending(cryp->tx);
 164
 165         return 0;
 166 }
 167
 168 static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx)
 169 {
 170         struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 171         struct starfive_cryp_dev *cryp = ctx->cryp;
 172         int ret = 0;
 173
 174         rctx->csr.hash.v = 0;
 175         rctx->csr.hash.reset = 1;
 176         writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
 177
 178         if (starfive_hash_wait_busy(ctx))
 179                 return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n");
 180
 181         rctx->csr.hash.v = 0;
 182         rctx->csr.hash.mode = ctx->hash_mode;
 183         rctx->csr.hash.ie = 1;
 184
 185         if (ctx->is_hmac) {
 186                 ret = starfive_hash_hmac_key(ctx);
 187                 if (ret)
 188                         return ret;
 189         } else {
 190                 rctx->csr.hash.start = 1;
 191                 rctx->csr.hash.firstb = 1;
 192                 writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
 193         }
 194
 195         return starfive_hash_xmit_dma(ctx);
 196 }
 197
 198 static int starfive_hash_copy_hash(struct ahash_request *req)
 199 {
 200         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 201         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
 202         int count, *data;
 203         int mlen;
 204
 205         if (!req->result)
 206                 return 0;
 207
 208         mlen = rctx->digsize / sizeof(u32);
 209         data = (u32 *)req->result;
 210
 211         for (count = 0; count < mlen; count++)
 212                 put_unaligned(readl(ctx->cryp->base + STARFIVE_HASH_SHARDR),
 213                               &data[count]);
 214
 215         return 0;
 216 }
 217
 218 void starfive_hash_done_task(unsigned long param)
 219 {
 220         struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
 221         int err = cryp->err;
 222
 223         if (!err)
 224                 err = starfive_hash_copy_hash(cryp->req.hreq);
 225
 226         /* Reset to clear hash_done in irq register*/
 227         writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR);
 228
 229         crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err);
 230 }
 231
 232 static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align)
 233 {
 234         int len = 0;
 235
 236         if (!total)
 237                 return 0;
 238
 239         if (!IS_ALIGNED(total, align))
 240                 return -EINVAL;
 241
 242         while (sg) {
 243                 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
 244                         return -EINVAL;
 245
 246                 if (!IS_ALIGNED(sg->length, align))
 247                         return -EINVAL;
 248
 249                 len += sg->length;
 250                 sg = sg_next(sg);
 251         }
 252
 253         if (len != total)
 254                 return -EINVAL;
 255
 256         return 0;
 257 }
 258
 259 static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
 260 {
 261         struct ahash_request *req = container_of(areq, struct ahash_request,
 262                                                  base);
 263         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
 264         struct starfive_cryp_dev *cryp = ctx->cryp;
 265
 266         if (!cryp)
 267                 return -ENODEV;
 268
 269         return starfive_hash_xmit(ctx);
 270 }
 271
 272 static int starfive_hash_init(struct ahash_request *req)
 273 {
 274         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 275         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 276         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 277
 278         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 279         ahash_request_set_callback(&rctx->ahash_fbk_req,
 280                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 281                                    req->base.complete, req->base.data);
 282
 283         ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
 284                                 req->result, req->nbytes);
 285
 286         return crypto_ahash_init(&rctx->ahash_fbk_req);
 287 }
 288
 289 static int starfive_hash_update(struct ahash_request *req)
 290 {
 291         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 292         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 293         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 294
 295         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 296         ahash_request_set_callback(&rctx->ahash_fbk_req,
 297                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 298                                    req->base.complete, req->base.data);
 299
 300         ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
 301                                 req->result, req->nbytes);
 302
 303         return crypto_ahash_update(&rctx->ahash_fbk_req);
 304 }
 305
 306 static int starfive_hash_final(struct ahash_request *req)
 307 {
 308         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 309         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 310         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 311
 312         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 313         ahash_request_set_callback(&rctx->ahash_fbk_req,
 314                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 315                                    req->base.complete, req->base.data);
 316
 317         ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
 318                                 req->result, req->nbytes);
 319
 320         return crypto_ahash_final(&rctx->ahash_fbk_req);
 321 }
 322
 323 static int starfive_hash_finup(struct ahash_request *req)
 324 {
 325         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 326         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 327         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 328
 329         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 330         ahash_request_set_callback(&rctx->ahash_fbk_req,
 331                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 332                                    req->base.complete, req->base.data);
 333
 334         ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
 335                                 req->result, req->nbytes);
 336
 337         return crypto_ahash_finup(&rctx->ahash_fbk_req);
 338 }
 339
 340 static int starfive_hash_digest_fb(struct ahash_request *req)
 341 {
 342         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 343         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 344         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 345
 346         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 347         ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags,
 348                                    req->base.complete, req->base.data);
 349
 350         ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
 351                                 req->result, req->nbytes);
 352
 353         return crypto_ahash_digest(&rctx->ahash_fbk_req);
 354 }
 355
 356 static int starfive_hash_digest(struct ahash_request *req)
 357 {
 358         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 359         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 360         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 361         struct starfive_cryp_dev *cryp = ctx->cryp;
 362
 363         memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx));
 364
 365         cryp->req.hreq = req;
 366         rctx->total = req->nbytes;
 367         rctx->in_sg = req->src;
 368         rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
 369         rctx->digsize = crypto_ahash_digestsize(tfm);
 370         rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
 371         ctx->rctx = rctx;
 372
 373         if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))
 374                 return starfive_hash_digest_fb(req);
 375
 376         return crypto_transfer_hash_request_to_engine(cryp->engine, req);
 377 }
 378
 379 static int starfive_hash_export(struct ahash_request *req, void *out)
 380 {
 381         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 382         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 383         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 384
 385         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 386         ahash_request_set_callback(&rctx->ahash_fbk_req,
 387                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 388                                    req->base.complete, req->base.data);
 389
 390         return crypto_ahash_export(&rctx->ahash_fbk_req, out);
 391 }
 392
 393 static int starfive_hash_import(struct ahash_request *req, const void *in)
 394 {
 395         struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
 396         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 397         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
 398
 399         ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
 400         ahash_request_set_callback(&rctx->ahash_fbk_req,
 401                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
 402                                    req->base.complete, req->base.data);
 403
 404         return crypto_ahash_import(&rctx->ahash_fbk_req, in);
 405 }
 406
 407 static int starfive_hash_init_tfm(struct crypto_ahash *hash,
 408                                   const char *alg_name,
 409                                   unsigned int mode)
 410 {
 411         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 412
 413         ctx->cryp = starfive_cryp_find_dev(ctx);
 414
 415         if (!ctx->cryp)
 416                 return -ENODEV;
 417
 418         ctx->ahash_fbk = crypto_alloc_ahash(alg_name, 0,
 419                                             CRYPTO_ALG_NEED_FALLBACK);
 420
 421         if (IS_ERR(ctx->ahash_fbk))
 422                 return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->ahash_fbk),
 423                                      "starfive_hash: Could not load fallback driver.\n");
 424
 425         crypto_ahash_set_statesize(hash, crypto_ahash_statesize(ctx->ahash_fbk));
 426         crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) +
 427                                  crypto_ahash_reqsize(ctx->ahash_fbk));
 428
 429         ctx->keylen = 0;
 430         ctx->hash_mode = mode;
 431
 432         return 0;
 433 }
 434
 435 static void starfive_hash_exit_tfm(struct crypto_ahash *hash)
 436 {
 437         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 438
 439         crypto_free_ahash(ctx->ahash_fbk);
 440 }
 441
 442 static int starfive_hash_long_setkey(struct starfive_cryp_ctx *ctx,
 443                                      const u8 *key, unsigned int keylen,
 444                                      const char *alg_name)
 445 {
 446         struct crypto_wait wait;
 447         struct ahash_request *req;
 448         struct scatterlist sg;
 449         struct crypto_ahash *ahash_tfm;
 450         u8 *buf;
 451         int ret;
 452
 453         ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
 454         if (IS_ERR(ahash_tfm))
 455                 return PTR_ERR(ahash_tfm);
 456
 457         req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
 458         if (!req) {
 459                 ret = -ENOMEM;
 460                 goto err_free_ahash;
 461         }
 462
 463         crypto_init_wait(&wait);
 464         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 465                                    crypto_req_done, &wait);
 466         crypto_ahash_clear_flags(ahash_tfm, ~0);
 467
 468         buf = kzalloc(keylen + STARFIVE_HASH_BUFLEN, GFP_KERNEL);
 469         if (!buf) {
 470                 ret = -ENOMEM;
 471                 goto err_free_req;
 472         }
 473
 474         memcpy(buf, key, keylen);
 475         sg_init_one(&sg, buf, keylen);
 476         ahash_request_set_crypt(req, &sg, ctx->key, keylen);
 477
 478         ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
 479
 480         kfree(buf);
 481 err_free_req:
 482         ahash_request_free(req);
 483 err_free_ahash:
 484         crypto_free_ahash(ahash_tfm);
 485         return ret;
 486 }
 487
 488 static int starfive_hash_setkey(struct crypto_ahash *hash,
 489                                 const u8 *key, unsigned int keylen)
 490 {
 491         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 492         unsigned int digestsize = crypto_ahash_digestsize(hash);
 493         unsigned int blocksize = crypto_ahash_blocksize(hash);
 494         const char *alg_name;
 495
 496         crypto_ahash_setkey(ctx->ahash_fbk, key, keylen);
 497
 498         if (keylen <= blocksize) {
 499                 memcpy(ctx->key, key, keylen);
 500                 ctx->keylen = keylen;
 501                 return 0;
 502         }
 503
 504         ctx->keylen = digestsize;
 505
 506         switch (digestsize) {
 507         case SHA224_DIGEST_SIZE:
 508                 alg_name = "sha224-starfive";
 509                 break;
 510         case SHA256_DIGEST_SIZE:
 511                 if (ctx->hash_mode == STARFIVE_HASH_SM3)
 512                         alg_name = "sm3-starfive";
 513                 else
 514                         alg_name = "sha256-starfive";
 515                 break;
 516         case SHA384_DIGEST_SIZE:
 517                 alg_name = "sha384-starfive";
 518                 break;
 519         case SHA512_DIGEST_SIZE:
 520                 alg_name = "sha512-starfive";
 521                 break;
 522         default:
 523                 return -EINVAL;
 524         }
 525
 526         return starfive_hash_long_setkey(ctx, key, keylen, alg_name);
 527 }
 528
 529 static int starfive_sha224_init_tfm(struct crypto_ahash *hash)
 530 {
 531         return starfive_hash_init_tfm(hash, "sha224-generic",
 532                                       STARFIVE_HASH_SHA224);
 533 }
 534
 535 static int starfive_sha256_init_tfm(struct crypto_ahash *hash)
 536 {
 537         return starfive_hash_init_tfm(hash, "sha256-generic",
 538                                       STARFIVE_HASH_SHA256);
 539 }
 540
 541 static int starfive_sha384_init_tfm(struct crypto_ahash *hash)
 542 {
 543         return starfive_hash_init_tfm(hash, "sha384-generic",
 544                                       STARFIVE_HASH_SHA384);
 545 }
 546
 547 static int starfive_sha512_init_tfm(struct crypto_ahash *hash)
 548 {
 549         return starfive_hash_init_tfm(hash, "sha512-generic",
 550                                       STARFIVE_HASH_SHA512);
 551 }
 552
 553 static int starfive_sm3_init_tfm(struct crypto_ahash *hash)
 554 {
 555         return starfive_hash_init_tfm(hash, "sm3-generic",
 556                                       STARFIVE_HASH_SM3);
 557 }
 558
 559 static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash)
 560 {
 561         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 562
 563         ctx->is_hmac = true;
 564
 565         return starfive_hash_init_tfm(hash, "hmac(sha224-generic)",
 566                                       STARFIVE_HASH_SHA224);
 567 }
 568
 569 static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash)
 570 {
 571         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 572
 573         ctx->is_hmac = true;
 574
 575         return starfive_hash_init_tfm(hash, "hmac(sha256-generic)",
 576                                       STARFIVE_HASH_SHA256);
 577 }
 578
 579 static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash)
 580 {
 581         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 582
 583         ctx->is_hmac = true;
 584
 585         return starfive_hash_init_tfm(hash, "hmac(sha384-generic)",
 586                                       STARFIVE_HASH_SHA384);
 587 }
 588
 589 static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash)
 590 {
 591         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 592
 593         ctx->is_hmac = true;
 594
 595         return starfive_hash_init_tfm(hash, "hmac(sha512-generic)",
 596                                       STARFIVE_HASH_SHA512);
 597 }
 598
 599 static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash)
 600 {
 601         struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 602
 603         ctx->is_hmac = true;
 604
 605         return starfive_hash_init_tfm(hash, "hmac(sm3-generic)",
 606                                       STARFIVE_HASH_SM3);
 607 }
 608
 609 static struct ahash_engine_alg algs_sha2_sm3[] = {
 610 {
 611         .base.init     = starfive_hash_init,
 612         .base.update   = starfive_hash_update,
 613         .base.final    = starfive_hash_final,
 614         .base.finup    = starfive_hash_finup,
 615         .base.digest   = starfive_hash_digest,
 616         .base.export   = starfive_hash_export,
 617         .base.import   = starfive_hash_import,
 618         .base.init_tfm = starfive_sha224_init_tfm,
 619         .base.exit_tfm = starfive_hash_exit_tfm,
 620         .base.halg = {
 621                 .digestsize = SHA224_DIGEST_SIZE,
 622                 .statesize  = sizeof(struct sha256_state),
 623                 .base = {
 624                         .cra_name               = "sha224",
 625                         .cra_driver_name        = "sha224-starfive",
 626                         .cra_priority           = 200,
 627                         .cra_flags              = CRYPTO_ALG_ASYNC |
 628                                                   CRYPTO_ALG_TYPE_AHASH |
 629                                                   CRYPTO_ALG_NEED_FALLBACK,
 630                         .cra_blocksize          = SHA224_BLOCK_SIZE,
 631                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 632                         .cra_module             = THIS_MODULE,
 633                 }
 634         },
 635         .op = {
 636                 .do_one_request = starfive_hash_one_request,
 637         },
 638 }, {
 639         .base.init     = starfive_hash_init,
 640         .base.update   = starfive_hash_update,
 641         .base.final    = starfive_hash_final,
 642         .base.finup    = starfive_hash_finup,
 643         .base.digest   = starfive_hash_digest,
 644         .base.export   = starfive_hash_export,
 645         .base.import   = starfive_hash_import,
 646         .base.init_tfm = starfive_hmac_sha224_init_tfm,
 647         .base.exit_tfm = starfive_hash_exit_tfm,
 648         .base.setkey   = starfive_hash_setkey,
 649         .base.halg = {
 650                 .digestsize = SHA224_DIGEST_SIZE,
 651                 .statesize  = sizeof(struct sha256_state),
 652                 .base = {
 653                         .cra_name               = "hmac(sha224)",
 654                         .cra_driver_name        = "sha224-hmac-starfive",
 655                         .cra_priority           = 200,
 656                         .cra_flags              = CRYPTO_ALG_ASYNC |
 657                                                   CRYPTO_ALG_TYPE_AHASH |
 658                                                   CRYPTO_ALG_NEED_FALLBACK,
 659                         .cra_blocksize          = SHA224_BLOCK_SIZE,
 660                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 661                         .cra_module             = THIS_MODULE,
 662                 }
 663         },
 664         .op = {
 665                 .do_one_request = starfive_hash_one_request,
 666         },
 667 }, {
 668         .base.init     = starfive_hash_init,
 669         .base.update   = starfive_hash_update,
 670         .base.final    = starfive_hash_final,
 671         .base.finup    = starfive_hash_finup,
 672         .base.digest   = starfive_hash_digest,
 673         .base.export   = starfive_hash_export,
 674         .base.import   = starfive_hash_import,
 675         .base.init_tfm = starfive_sha256_init_tfm,
 676         .base.exit_tfm = starfive_hash_exit_tfm,
 677         .base.halg = {
 678                 .digestsize = SHA256_DIGEST_SIZE,
 679                 .statesize  = sizeof(struct sha256_state),
 680                 .base = {
 681                         .cra_name               = "sha256",
 682                         .cra_driver_name        = "sha256-starfive",
 683                         .cra_priority           = 200,
 684                         .cra_flags              = CRYPTO_ALG_ASYNC |
 685                                                   CRYPTO_ALG_TYPE_AHASH |
 686                                                   CRYPTO_ALG_NEED_FALLBACK,
 687                         .cra_blocksize          = SHA256_BLOCK_SIZE,
 688                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 689                         .cra_module             = THIS_MODULE,
 690                 }
 691         },
 692         .op = {
 693                 .do_one_request = starfive_hash_one_request,
 694         },
 695 }, {
 696         .base.init     = starfive_hash_init,
 697         .base.update   = starfive_hash_update,
 698         .base.final    = starfive_hash_final,
 699         .base.finup    = starfive_hash_finup,
 700         .base.digest   = starfive_hash_digest,
 701         .base.export   = starfive_hash_export,
 702         .base.import   = starfive_hash_import,
 703         .base.init_tfm = starfive_hmac_sha256_init_tfm,
 704         .base.exit_tfm = starfive_hash_exit_tfm,
 705         .base.setkey   = starfive_hash_setkey,
 706         .base.halg = {
 707                 .digestsize = SHA256_DIGEST_SIZE,
 708                 .statesize  = sizeof(struct sha256_state),
 709                 .base = {
 710                         .cra_name               = "hmac(sha256)",
 711                         .cra_driver_name        = "sha256-hmac-starfive",
 712                         .cra_priority           = 200,
 713                         .cra_flags              = CRYPTO_ALG_ASYNC |
 714                                                   CRYPTO_ALG_TYPE_AHASH |
 715                                                   CRYPTO_ALG_NEED_FALLBACK,
 716                         .cra_blocksize          = SHA256_BLOCK_SIZE,
 717                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 718                         .cra_module             = THIS_MODULE,
 719                 }
 720         },
 721         .op = {
 722                 .do_one_request = starfive_hash_one_request,
 723         },
 724 }, {
 725         .base.init     = starfive_hash_init,
 726         .base.update   = starfive_hash_update,
 727         .base.final    = starfive_hash_final,
 728         .base.finup    = starfive_hash_finup,
 729         .base.digest   = starfive_hash_digest,
 730         .base.export   = starfive_hash_export,
 731         .base.import   = starfive_hash_import,
 732         .base.init_tfm = starfive_sha384_init_tfm,
 733         .base.exit_tfm = starfive_hash_exit_tfm,
 734         .base.halg = {
 735                 .digestsize = SHA384_DIGEST_SIZE,
 736                 .statesize  = sizeof(struct sha512_state),
 737                 .base = {
 738                         .cra_name               = "sha384",
 739                         .cra_driver_name        = "sha384-starfive",
 740                         .cra_priority           = 200,
 741                         .cra_flags              = CRYPTO_ALG_ASYNC |
 742                                                   CRYPTO_ALG_TYPE_AHASH |
 743                                                   CRYPTO_ALG_NEED_FALLBACK,
 744                         .cra_blocksize          = SHA384_BLOCK_SIZE,
 745                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 746                         .cra_module             = THIS_MODULE,
 747                 }
 748         },
 749         .op = {
 750                 .do_one_request = starfive_hash_one_request,
 751         },
 752 }, {
 753         .base.init     = starfive_hash_init,
 754         .base.update   = starfive_hash_update,
 755         .base.final    = starfive_hash_final,
 756         .base.finup    = starfive_hash_finup,
 757         .base.digest   = starfive_hash_digest,
 758         .base.export   = starfive_hash_export,
 759         .base.import   = starfive_hash_import,
 760         .base.init_tfm = starfive_hmac_sha384_init_tfm,
 761         .base.exit_tfm = starfive_hash_exit_tfm,
 762         .base.setkey   = starfive_hash_setkey,
 763         .base.halg = {
 764                 .digestsize = SHA384_DIGEST_SIZE,
 765                 .statesize  = sizeof(struct sha512_state),
 766                 .base = {
 767                         .cra_name               = "hmac(sha384)",
 768                         .cra_driver_name        = "sha384-hmac-starfive",
 769                         .cra_priority           = 200,
 770                         .cra_flags              = CRYPTO_ALG_ASYNC |
 771                                                   CRYPTO_ALG_TYPE_AHASH |
 772                                                   CRYPTO_ALG_NEED_FALLBACK,
 773                         .cra_blocksize          = SHA384_BLOCK_SIZE,
 774                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 775                         .cra_module             = THIS_MODULE,
 776                 }
 777         },
 778         .op = {
 779                 .do_one_request = starfive_hash_one_request,
 780         },
 781 }, {
 782         .base.init     = starfive_hash_init,
 783         .base.update   = starfive_hash_update,
 784         .base.final    = starfive_hash_final,
 785         .base.finup    = starfive_hash_finup,
 786         .base.digest   = starfive_hash_digest,
 787         .base.export   = starfive_hash_export,
 788         .base.import   = starfive_hash_import,
 789         .base.init_tfm = starfive_sha512_init_tfm,
 790         .base.exit_tfm = starfive_hash_exit_tfm,
 791         .base.halg = {
 792                 .digestsize = SHA512_DIGEST_SIZE,
 793                 .statesize  = sizeof(struct sha512_state),
 794                 .base = {
 795                         .cra_name               = "sha512",
 796                         .cra_driver_name        = "sha512-starfive",
 797                         .cra_priority           = 200,
 798                         .cra_flags              = CRYPTO_ALG_ASYNC |
 799                                                   CRYPTO_ALG_TYPE_AHASH |
 800                                                   CRYPTO_ALG_NEED_FALLBACK,
 801                         .cra_blocksize          = SHA512_BLOCK_SIZE,
 802                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 803                         .cra_module             = THIS_MODULE,
 804                 }
 805         },
 806         .op = {
 807                 .do_one_request = starfive_hash_one_request,
 808         },
 809 }, {
 810         .base.init     = starfive_hash_init,
 811         .base.update   = starfive_hash_update,
 812         .base.final    = starfive_hash_final,
 813         .base.finup    = starfive_hash_finup,
 814         .base.digest   = starfive_hash_digest,
 815         .base.export   = starfive_hash_export,
 816         .base.import   = starfive_hash_import,
 817         .base.init_tfm = starfive_hmac_sha512_init_tfm,
 818         .base.exit_tfm = starfive_hash_exit_tfm,
 819         .base.setkey   = starfive_hash_setkey,
 820         .base.halg = {
 821                 .digestsize = SHA512_DIGEST_SIZE,
 822                 .statesize  = sizeof(struct sha512_state),
 823                 .base = {
 824                         .cra_name               = "hmac(sha512)",
 825                         .cra_driver_name        = "sha512-hmac-starfive",
 826                         .cra_priority           = 200,
 827                         .cra_flags              = CRYPTO_ALG_ASYNC |
 828                                                   CRYPTO_ALG_TYPE_AHASH |
 829                                                   CRYPTO_ALG_NEED_FALLBACK,
 830                         .cra_blocksize          = SHA512_BLOCK_SIZE,
 831                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 832                         .cra_module             = THIS_MODULE,
 833                 }
 834         },
 835         .op = {
 836                 .do_one_request = starfive_hash_one_request,
 837         },
 838 }, {
 839         .base.init     = starfive_hash_init,
 840         .base.update   = starfive_hash_update,
 841         .base.final    = starfive_hash_final,
 842         .base.finup    = starfive_hash_finup,
 843         .base.digest   = starfive_hash_digest,
 844         .base.export   = starfive_hash_export,
 845         .base.import   = starfive_hash_import,
 846         .base.init_tfm = starfive_sm3_init_tfm,
 847         .base.exit_tfm = starfive_hash_exit_tfm,
 848         .base.halg = {
 849                 .digestsize = SM3_DIGEST_SIZE,
 850                 .statesize  = sizeof(struct sm3_state),
 851                 .base = {
 852                         .cra_name               = "sm3",
 853                         .cra_driver_name        = "sm3-starfive",
 854                         .cra_priority           = 200,
 855                         .cra_flags              = CRYPTO_ALG_ASYNC |
 856                                                   CRYPTO_ALG_TYPE_AHASH |
 857                                                   CRYPTO_ALG_NEED_FALLBACK,
 858                         .cra_blocksize          = SM3_BLOCK_SIZE,
 859                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 860                         .cra_module             = THIS_MODULE,
 861                 }
 862         },
 863         .op = {
 864                 .do_one_request = starfive_hash_one_request,
 865         },
 866 }, {
 867         .base.init        = starfive_hash_init,
 868         .base.update      = starfive_hash_update,
 869         .base.final       = starfive_hash_final,
 870         .base.finup       = starfive_hash_finup,
 871         .base.digest      = starfive_hash_digest,
 872         .base.export      = starfive_hash_export,
 873         .base.import      = starfive_hash_import,
 874         .base.init_tfm = starfive_hmac_sm3_init_tfm,
 875         .base.exit_tfm = starfive_hash_exit_tfm,
 876         .base.setkey      = starfive_hash_setkey,
 877         .base.halg = {
 878                 .digestsize = SM3_DIGEST_SIZE,
 879                 .statesize  = sizeof(struct sm3_state),
 880                 .base = {
 881                         .cra_name               = "hmac(sm3)",
 882                         .cra_driver_name        = "sm3-hmac-starfive",
 883                         .cra_priority           = 200,
 884                         .cra_flags              = CRYPTO_ALG_ASYNC |
 885                                                   CRYPTO_ALG_TYPE_AHASH |
 886                                                   CRYPTO_ALG_NEED_FALLBACK,
 887                         .cra_blocksize          = SM3_BLOCK_SIZE,
 888                         .cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
 889                         .cra_module             = THIS_MODULE,
 890                 }
 891         },
 892         .op = {
 893                 .do_one_request = starfive_hash_one_request,
 894         },
 895 },
 896 };
 897
 898 int starfive_hash_register_algs(void)
 899 {
 900         return crypto_engine_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
 901 }
 902
 903 void starfive_hash_unregister_algs(void)
 904 {
 905         crypto_engine_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
 906 }