drivers/crypto/img-hash.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2014 Imagination Technologies
   4  * Authors:  Will Thomas, James Hartley
   5  *
   6  *      Interface structure taken from omap-sham driver
   7  */
   8
   9 #include <linux/clk.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/of_device.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/scatterlist.h>
  19
  20 #include <crypto/internal/hash.h>
  21 #include <crypto/md5.h>
  22 #include <crypto/sha.h>
  23
  24 #define CR_RESET                        0
  25 #define CR_RESET_SET                    1
  26 #define CR_RESET_UNSET                  0
  27
  28 #define CR_MESSAGE_LENGTH_H             0x4
  29 #define CR_MESSAGE_LENGTH_L             0x8
  30
  31 #define CR_CONTROL                      0xc
  32 #define CR_CONTROL_BYTE_ORDER_3210      0
  33 #define CR_CONTROL_BYTE_ORDER_0123      1
  34 #define CR_CONTROL_BYTE_ORDER_2310      2
  35 #define CR_CONTROL_BYTE_ORDER_1032      3
  36 #define CR_CONTROL_BYTE_ORDER_SHIFT     8
  37 #define CR_CONTROL_ALGO_MD5     0
  38 #define CR_CONTROL_ALGO_SHA1    1
  39 #define CR_CONTROL_ALGO_SHA224  2
  40 #define CR_CONTROL_ALGO_SHA256  3
  41
  42 #define CR_INTSTAT                      0x10
  43 #define CR_INTENAB                      0x14
  44 #define CR_INTCLEAR                     0x18
  45 #define CR_INT_RESULTS_AVAILABLE        BIT(0)
  46 #define CR_INT_NEW_RESULTS_SET          BIT(1)
  47 #define CR_INT_RESULT_READ_ERR          BIT(2)
  48 #define CR_INT_MESSAGE_WRITE_ERROR      BIT(3)
  49 #define CR_INT_STATUS                   BIT(8)
  50
  51 #define CR_RESULT_QUEUE         0x1c
  52 #define CR_RSD0                         0x40
  53 #define CR_CORE_REV                     0x50
  54 #define CR_CORE_DES1            0x60
  55 #define CR_CORE_DES2            0x70
  56
  57 #define DRIVER_FLAGS_BUSY               BIT(0)
  58 #define DRIVER_FLAGS_FINAL              BIT(1)
  59 #define DRIVER_FLAGS_DMA_ACTIVE         BIT(2)
  60 #define DRIVER_FLAGS_OUTPUT_READY       BIT(3)
  61 #define DRIVER_FLAGS_INIT               BIT(4)
  62 #define DRIVER_FLAGS_CPU                BIT(5)
  63 #define DRIVER_FLAGS_DMA_READY          BIT(6)
  64 #define DRIVER_FLAGS_ERROR              BIT(7)
  65 #define DRIVER_FLAGS_SG                 BIT(8)
  66 #define DRIVER_FLAGS_SHA1               BIT(18)
  67 #define DRIVER_FLAGS_SHA224             BIT(19)
  68 #define DRIVER_FLAGS_SHA256             BIT(20)
  69 #define DRIVER_FLAGS_MD5                BIT(21)
  70
  71 #define IMG_HASH_QUEUE_LENGTH           20
  72 #define IMG_HASH_DMA_BURST              4
  73 #define IMG_HASH_DMA_THRESHOLD          64
  74
  75 #ifdef __LITTLE_ENDIAN
  76 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_3210
  77 #else
  78 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_0123
  79 #endif
  80
  81 struct img_hash_dev;
  82
  83 struct img_hash_request_ctx {
  84         struct img_hash_dev     *hdev;
  85         u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
  86         unsigned long           flags;
  87         size_t                  digsize;
  88
  89         dma_addr_t              dma_addr;
  90         size_t                  dma_ct;
  91
  92         /* sg root */
  93         struct scatterlist      *sgfirst;
  94         /* walk state */
  95         struct scatterlist      *sg;
  96         size_t                  nents;
  97         size_t                  offset;
  98         unsigned int            total;
  99         size_t                  sent;
 100
 101         unsigned long           op;
 102
 103         size_t                  bufcnt;
 104         struct ahash_request    fallback_req;
 105
 106         /* Zero length buffer must remain last member of struct */
 107         u8 buffer[] __aligned(sizeof(u32));
 108 };
 109
 110 struct img_hash_ctx {
 111         struct img_hash_dev     *hdev;
 112         unsigned long           flags;
 113         struct crypto_ahash     *fallback;
 114 };
 115
 116 struct img_hash_dev {
 117         struct list_head        list;
 118         struct device           *dev;
 119         struct clk              *hash_clk;
 120         struct clk              *sys_clk;
 121         void __iomem            *io_base;
 122
 123         phys_addr_t             bus_addr;
 124         void __iomem            *cpu_addr;
 125
 126         spinlock_t              lock;
 127         int                     err;
 128         struct tasklet_struct   done_task;
 129         struct tasklet_struct   dma_task;
 130
 131         unsigned long           flags;
 132         struct crypto_queue     queue;
 133         struct ahash_request    *req;
 134
 135         struct dma_chan         *dma_lch;
 136 };
 137
 138 struct img_hash_drv {
 139         struct list_head dev_list;
 140         spinlock_t lock;
 141 };
 142
 143 static struct img_hash_drv img_hash = {
 144         .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
 145         .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
 146 };
 147
 148 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
 149 {
 150         return readl_relaxed(hdev->io_base + offset);
 151 }
 152
 153 static inline void img_hash_write(struct img_hash_dev *hdev,
 154                                   u32 offset, u32 value)
 155 {
 156         writel_relaxed(value, hdev->io_base + offset);
 157 }
 158
 159 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
 160 {
 161         return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
 162 }
 163
 164 static void img_hash_start(struct img_hash_dev *hdev, bool dma)
 165 {
 166         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 167         u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
 168
 169         if (ctx->flags & DRIVER_FLAGS_MD5)
 170                 cr |= CR_CONTROL_ALGO_MD5;
 171         else if (ctx->flags & DRIVER_FLAGS_SHA1)
 172                 cr |= CR_CONTROL_ALGO_SHA1;
 173         else if (ctx->flags & DRIVER_FLAGS_SHA224)
 174                 cr |= CR_CONTROL_ALGO_SHA224;
 175         else if (ctx->flags & DRIVER_FLAGS_SHA256)
 176                 cr |= CR_CONTROL_ALGO_SHA256;
 177         dev_dbg(hdev->dev, "Starting hash process\n");
 178         img_hash_write(hdev, CR_CONTROL, cr);
 179
 180         /*
 181          * The hardware block requires two cycles between writing the control
 182          * register and writing the first word of data in non DMA mode, to
 183          * ensure the first data write is not grouped in burst with the control
 184          * register write a read is issued to 'flush' the bus.
 185          */
 186         if (!dma)
 187                 img_hash_read(hdev, CR_CONTROL);
 188 }
 189
 190 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
 191                              size_t length, int final)
 192 {
 193         u32 count, len32;
 194         const u32 *buffer = (const u32 *)buf;
 195
 196         dev_dbg(hdev->dev, "xmit_cpu:  length: %zu bytes\n", length);
 197
 198         if (final)
 199                 hdev->flags |= DRIVER_FLAGS_FINAL;
 200
 201         len32 = DIV_ROUND_UP(length, sizeof(u32));
 202
 203         for (count = 0; count < len32; count++)
 204                 writel_relaxed(buffer[count], hdev->cpu_addr);
 205
 206         return -EINPROGRESS;
 207 }
 208
 209 static void img_hash_dma_callback(void *data)
 210 {
 211         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 212         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 213
 214         if (ctx->bufcnt) {
 215                 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
 216                 ctx->bufcnt = 0;
 217         }
 218         if (ctx->sg)
 219                 tasklet_schedule(&hdev->dma_task);
 220 }
 221
 222 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
 223 {
 224         struct dma_async_tx_descriptor *desc;
 225         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 226
 227         ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 228         if (ctx->dma_ct == 0) {
 229                 dev_err(hdev->dev, "Invalid DMA sg\n");
 230                 hdev->err = -EINVAL;
 231                 return -EINVAL;
 232         }
 233
 234         desc = dmaengine_prep_slave_sg(hdev->dma_lch,
 235                                        sg,
 236                                        ctx->dma_ct,
 237                                        DMA_MEM_TO_DEV,
 238                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 239         if (!desc) {
 240                 dev_err(hdev->dev, "Null DMA descriptor\n");
 241                 hdev->err = -EINVAL;
 242                 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 243                 return -EINVAL;
 244         }
 245         desc->callback = img_hash_dma_callback;
 246         desc->callback_param = hdev;
 247         dmaengine_submit(desc);
 248         dma_async_issue_pending(hdev->dma_lch);
 249
 250         return 0;
 251 }
 252
 253 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
 254 {
 255         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 256
 257         ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
 258                                         ctx->buffer, hdev->req->nbytes);
 259
 260         ctx->total = hdev->req->nbytes;
 261         ctx->bufcnt = 0;
 262
 263         hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
 264
 265         img_hash_start(hdev, false);
 266
 267         return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
 268 }
 269
 270 static int img_hash_finish(struct ahash_request *req)
 271 {
 272         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 273
 274         if (!req->result)
 275                 return -EINVAL;
 276
 277         memcpy(req->result, ctx->digest, ctx->digsize);
 278
 279         return 0;
 280 }
 281
 282 static void img_hash_copy_hash(struct ahash_request *req)
 283 {
 284         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 285         u32 *hash = (u32 *)ctx->digest;
 286         int i;
 287
 288         for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
 289                 hash[i] = img_hash_read_result_queue(ctx->hdev);
 290 }
 291
 292 static void img_hash_finish_req(struct ahash_request *req, int err)
 293 {
 294         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 295         struct img_hash_dev *hdev =  ctx->hdev;
 296
 297         if (!err) {
 298                 img_hash_copy_hash(req);
 299                 if (DRIVER_FLAGS_FINAL & hdev->flags)
 300                         err = img_hash_finish(req);
 301         } else {
 302                 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
 303                 ctx->flags |= DRIVER_FLAGS_ERROR;
 304         }
 305
 306         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
 307                 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
 308
 309         if (req->base.complete)
 310                 req->base.complete(&req->base, err);
 311 }
 312
 313 static int img_hash_write_via_dma(struct img_hash_dev *hdev)
 314 {
 315         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 316
 317         img_hash_start(hdev, true);
 318
 319         dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
 320
 321         if (!ctx->total)
 322                 hdev->flags |= DRIVER_FLAGS_FINAL;
 323
 324         hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
 325
 326         tasklet_schedule(&hdev->dma_task);
 327
 328         return -EINPROGRESS;
 329 }
 330
 331 static int img_hash_dma_init(struct img_hash_dev *hdev)
 332 {
 333         struct dma_slave_config dma_conf;
 334         int err;
 335
 336         hdev->dma_lch = dma_request_chan(hdev->dev, "tx");
 337         if (IS_ERR(hdev->dma_lch)) {
 338                 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
 339                 return PTR_ERR(hdev->dma_lch);
 340         }
 341         dma_conf.direction = DMA_MEM_TO_DEV;
 342         dma_conf.dst_addr = hdev->bus_addr;
 343         dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 344         dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;
 345         dma_conf.device_fc = false;
 346
 347         err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);
 348         if (err) {
 349                 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
 350                 dma_release_channel(hdev->dma_lch);
 351                 return err;
 352         }
 353
 354         return 0;
 355 }
 356
 357 static void img_hash_dma_task(unsigned long d)
 358 {
 359         struct img_hash_dev *hdev = (struct img_hash_dev *)d;
 360         struct img_hash_request_ctx *ctx;
 361         u8 *addr;
 362         size_t nbytes, bleft, wsend, len, tbc;
 363         struct scatterlist tsg;
 364
 365         if (!hdev->req)
 366                 return;
 367
 368         ctx = ahash_request_ctx(hdev->req);
 369         if (!ctx->sg)
 370                 return;
 371
 372         addr = sg_virt(ctx->sg);
 373         nbytes = ctx->sg->length - ctx->offset;
 374
 375         /*
 376          * The hash accelerator does not support a data valid mask. This means
 377          * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
 378          * padding bytes in the last word written by that dma would erroneously
 379          * be included in the hash. To avoid this we round down the transfer,
 380          * and add the excess to the start of the next dma. It does not matter
 381          * that the final dma may not be a multiple of 4 bytes as the hashing
 382          * block is programmed to accept the correct number of bytes.
 383          */
 384
 385         bleft = nbytes % 4;
 386         wsend = (nbytes / 4);
 387
 388         if (wsend) {
 389                 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
 390                 if (img_hash_xmit_dma(hdev, &tsg)) {
 391                         dev_err(hdev->dev, "DMA failed, falling back to CPU");
 392                         ctx->flags |= DRIVER_FLAGS_CPU;
 393                         hdev->err = 0;
 394                         img_hash_xmit_cpu(hdev, addr + ctx->offset,
 395                                           wsend * 4, 0);
 396                         ctx->sent += wsend * 4;
 397                         wsend = 0;
 398                 } else {
 399                         ctx->sent += wsend * 4;
 400                 }
 401         }
 402
 403         if (bleft) {
 404                 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 405                                                  ctx->buffer, bleft, ctx->sent);
 406                 tbc = 0;
 407                 ctx->sg = sg_next(ctx->sg);
 408                 while (ctx->sg && (ctx->bufcnt < 4)) {
 409                         len = ctx->sg->length;
 410                         if (likely(len > (4 - ctx->bufcnt)))
 411                                 len = 4 - ctx->bufcnt;
 412                         tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 413                                                  ctx->buffer + ctx->bufcnt, len,
 414                                         ctx->sent + ctx->bufcnt);
 415                         ctx->bufcnt += tbc;
 416                         if (tbc >= ctx->sg->length) {
 417                                 ctx->sg = sg_next(ctx->sg);
 418                                 tbc = 0;
 419                         }
 420                 }
 421
 422                 ctx->sent += ctx->bufcnt;
 423                 ctx->offset = tbc;
 424
 425                 if (!wsend)
 426                         img_hash_dma_callback(hdev);
 427         } else {
 428                 ctx->offset = 0;
 429                 ctx->sg = sg_next(ctx->sg);
 430         }
 431 }
 432
 433 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
 434 {
 435         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 436
 437         if (ctx->flags & DRIVER_FLAGS_SG)
 438                 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
 439
 440         return 0;
 441 }
 442
 443 static int img_hash_process_data(struct img_hash_dev *hdev)
 444 {
 445         struct ahash_request *req = hdev->req;
 446         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 447         int err = 0;
 448
 449         ctx->bufcnt = 0;
 450
 451         if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
 452                 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
 453                         req->nbytes);
 454                 err = img_hash_write_via_dma(hdev);
 455         } else {
 456                 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
 457                         req->nbytes);
 458                 err = img_hash_write_via_cpu(hdev);
 459         }
 460         return err;
 461 }
 462
 463 static int img_hash_hw_init(struct img_hash_dev *hdev)
 464 {
 465         unsigned long long nbits;
 466         u32 u, l;
 467
 468         img_hash_write(hdev, CR_RESET, CR_RESET_SET);
 469         img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
 470         img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
 471
 472         nbits = (u64)hdev->req->nbytes << 3;
 473         u = nbits >> 32;
 474         l = nbits;
 475         img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
 476         img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
 477
 478         if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
 479                 hdev->flags |= DRIVER_FLAGS_INIT;
 480                 hdev->err = 0;
 481         }
 482         dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
 483         return 0;
 484 }
 485
 486 static int img_hash_init(struct ahash_request *req)
 487 {
 488         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 489         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 490         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 491
 492         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 493         rctx->fallback_req.base.flags = req->base.flags
 494                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 495
 496         return crypto_ahash_init(&rctx->fallback_req);
 497 }
 498
 499 static int img_hash_handle_queue(struct img_hash_dev *hdev,
 500                                  struct ahash_request *req)
 501 {
 502         struct crypto_async_request *async_req, *backlog;
 503         struct img_hash_request_ctx *ctx;
 504         unsigned long flags;
 505         int err = 0, res = 0;
 506
 507         spin_lock_irqsave(&hdev->lock, flags);
 508
 509         if (req)
 510                 res = ahash_enqueue_request(&hdev->queue, req);
 511
 512         if (DRIVER_FLAGS_BUSY & hdev->flags) {
 513                 spin_unlock_irqrestore(&hdev->lock, flags);
 514                 return res;
 515         }
 516
 517         backlog = crypto_get_backlog(&hdev->queue);
 518         async_req = crypto_dequeue_request(&hdev->queue);
 519         if (async_req)
 520                 hdev->flags |= DRIVER_FLAGS_BUSY;
 521
 522         spin_unlock_irqrestore(&hdev->lock, flags);
 523
 524         if (!async_req)
 525                 return res;
 526
 527         if (backlog)
 528                 backlog->complete(backlog, -EINPROGRESS);
 529
 530         req = ahash_request_cast(async_req);
 531         hdev->req = req;
 532
 533         ctx = ahash_request_ctx(req);
 534
 535         dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
 536                  ctx->op, req->nbytes);
 537
 538         err = img_hash_hw_init(hdev);
 539
 540         if (!err)
 541                 err = img_hash_process_data(hdev);
 542
 543         if (err != -EINPROGRESS) {
 544                 /* done_task will not finish so do it here */
 545                 img_hash_finish_req(req, err);
 546         }
 547         return res;
 548 }
 549
 550 static int img_hash_update(struct ahash_request *req)
 551 {
 552         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 553         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 554         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 555
 556         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 557         rctx->fallback_req.base.flags = req->base.flags
 558                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 559         rctx->fallback_req.nbytes = req->nbytes;
 560         rctx->fallback_req.src = req->src;
 561
 562         return crypto_ahash_update(&rctx->fallback_req);
 563 }
 564
 565 static int img_hash_final(struct ahash_request *req)
 566 {
 567         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 568         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 569         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 570
 571         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 572         rctx->fallback_req.base.flags = req->base.flags
 573                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 574         rctx->fallback_req.result = req->result;
 575
 576         return crypto_ahash_final(&rctx->fallback_req);
 577 }
 578
 579 static int img_hash_finup(struct ahash_request *req)
 580 {
 581         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 582         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 583         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 584
 585         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 586         rctx->fallback_req.base.flags = req->base.flags
 587                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 588         rctx->fallback_req.nbytes = req->nbytes;
 589         rctx->fallback_req.src = req->src;
 590         rctx->fallback_req.result = req->result;
 591
 592         return crypto_ahash_finup(&rctx->fallback_req);
 593 }
 594
 595 static int img_hash_import(struct ahash_request *req, const void *in)
 596 {
 597         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 598         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 599         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 600
 601         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 602         rctx->fallback_req.base.flags = req->base.flags
 603                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 604
 605         return crypto_ahash_import(&rctx->fallback_req, in);
 606 }
 607
 608 static int img_hash_export(struct ahash_request *req, void *out)
 609 {
 610         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 611         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 612         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 613
 614         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 615         rctx->fallback_req.base.flags = req->base.flags
 616                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 617
 618         return crypto_ahash_export(&rctx->fallback_req, out);
 619 }
 620
 621 static int img_hash_digest(struct ahash_request *req)
 622 {
 623         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 624         struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
 625         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 626         struct img_hash_dev *hdev = NULL;
 627         struct img_hash_dev *tmp;
 628         int err;
 629
 630         spin_lock(&img_hash.lock);
 631         if (!tctx->hdev) {
 632                 list_for_each_entry(tmp, &img_hash.dev_list, list) {
 633                         hdev = tmp;
 634                         break;
 635                 }
 636                 tctx->hdev = hdev;
 637
 638         } else {
 639                 hdev = tctx->hdev;
 640         }
 641
 642         spin_unlock(&img_hash.lock);
 643         ctx->hdev = hdev;
 644         ctx->flags = 0;
 645         ctx->digsize = crypto_ahash_digestsize(tfm);
 646
 647         switch (ctx->digsize) {
 648         case SHA1_DIGEST_SIZE:
 649                 ctx->flags |= DRIVER_FLAGS_SHA1;
 650                 break;
 651         case SHA256_DIGEST_SIZE:
 652                 ctx->flags |= DRIVER_FLAGS_SHA256;
 653                 break;
 654         case SHA224_DIGEST_SIZE:
 655                 ctx->flags |= DRIVER_FLAGS_SHA224;
 656                 break;
 657         case MD5_DIGEST_SIZE:
 658                 ctx->flags |= DRIVER_FLAGS_MD5;
 659                 break;
 660         default:
 661                 return -EINVAL;
 662         }
 663
 664         ctx->bufcnt = 0;
 665         ctx->offset = 0;
 666         ctx->sent = 0;
 667         ctx->total = req->nbytes;
 668         ctx->sg = req->src;
 669         ctx->sgfirst = req->src;
 670         ctx->nents = sg_nents(ctx->sg);
 671
 672         err = img_hash_handle_queue(tctx->hdev, req);
 673
 674         return err;
 675 }
 676
 677 static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)
 678 {
 679         struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 680         int err = -ENOMEM;
 681
 682         ctx->fallback = crypto_alloc_ahash(alg_name, 0,
 683                                            CRYPTO_ALG_NEED_FALLBACK);
 684         if (IS_ERR(ctx->fallback)) {
 685                 pr_err("img_hash: Could not load fallback driver.\n");
 686                 err = PTR_ERR(ctx->fallback);
 687                 goto err;
 688         }
 689         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 690                                  sizeof(struct img_hash_request_ctx) +
 691                                  crypto_ahash_reqsize(ctx->fallback) +
 692                                  IMG_HASH_DMA_THRESHOLD);
 693
 694         return 0;
 695
 696 err:
 697         return err;
 698 }
 699
 700 static int img_hash_cra_md5_init(struct crypto_tfm *tfm)
 701 {
 702         return img_hash_cra_init(tfm, "md5-generic");
 703 }
 704
 705 static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)
 706 {
 707         return img_hash_cra_init(tfm, "sha1-generic");
 708 }
 709
 710 static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)
 711 {
 712         return img_hash_cra_init(tfm, "sha224-generic");
 713 }
 714
 715 static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)
 716 {
 717         return img_hash_cra_init(tfm, "sha256-generic");
 718 }
 719
 720 static void img_hash_cra_exit(struct crypto_tfm *tfm)
 721 {
 722         struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
 723
 724         crypto_free_ahash(tctx->fallback);
 725 }
 726
 727 static irqreturn_t img_irq_handler(int irq, void *dev_id)
 728 {
 729         struct img_hash_dev *hdev = dev_id;
 730         u32 reg;
 731
 732         reg = img_hash_read(hdev, CR_INTSTAT);
 733         img_hash_write(hdev, CR_INTCLEAR, reg);
 734
 735         if (reg & CR_INT_NEW_RESULTS_SET) {
 736                 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
 737                 if (DRIVER_FLAGS_BUSY & hdev->flags) {
 738                         hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
 739                         if (!(DRIVER_FLAGS_CPU & hdev->flags))
 740                                 hdev->flags |= DRIVER_FLAGS_DMA_READY;
 741                         tasklet_schedule(&hdev->done_task);
 742                 } else {
 743                         dev_warn(hdev->dev,
 744                                  "HASH interrupt when no active requests.\n");
 745                 }
 746         } else if (reg & CR_INT_RESULTS_AVAILABLE) {
 747                 dev_warn(hdev->dev,
 748                          "IRQ triggered before the hash had completed\n");
 749         } else if (reg & CR_INT_RESULT_READ_ERR) {
 750                 dev_warn(hdev->dev,
 751                          "Attempt to read from an empty result queue\n");
 752         } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
 753                 dev_warn(hdev->dev,
 754                          "Data written before the hardware was configured\n");
 755         }
 756         return IRQ_HANDLED;
 757 }
 758
 759 static struct ahash_alg img_algs[] = {
 760         {
 761                 .init = img_hash_init,
 762                 .update = img_hash_update,
 763                 .final = img_hash_final,
 764                 .finup = img_hash_finup,
 765                 .export = img_hash_export,
 766                 .import = img_hash_import,
 767                 .digest = img_hash_digest,
 768                 .halg = {
 769                         .digestsize = MD5_DIGEST_SIZE,
 770                         .statesize = sizeof(struct md5_state),
 771                         .base = {
 772                                 .cra_name = "md5",
 773                                 .cra_driver_name = "img-md5",
 774                                 .cra_priority = 300,
 775                                 .cra_flags =
 776                                 CRYPTO_ALG_ASYNC |
 777                                 CRYPTO_ALG_NEED_FALLBACK,
 778                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
 779                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 780                                 .cra_init = img_hash_cra_md5_init,
 781                                 .cra_exit = img_hash_cra_exit,
 782                                 .cra_module = THIS_MODULE,
 783                         }
 784                 }
 785         },
 786         {
 787                 .init = img_hash_init,
 788                 .update = img_hash_update,
 789                 .final = img_hash_final,
 790                 .finup = img_hash_finup,
 791                 .export = img_hash_export,
 792                 .import = img_hash_import,
 793                 .digest = img_hash_digest,
 794                 .halg = {
 795                         .digestsize = SHA1_DIGEST_SIZE,
 796                         .statesize = sizeof(struct sha1_state),
 797                         .base = {
 798                                 .cra_name = "sha1",
 799                                 .cra_driver_name = "img-sha1",
 800                                 .cra_priority = 300,
 801                                 .cra_flags =
 802                                 CRYPTO_ALG_ASYNC |
 803                                 CRYPTO_ALG_NEED_FALLBACK,
 804                                 .cra_blocksize = SHA1_BLOCK_SIZE,
 805                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 806                                 .cra_init = img_hash_cra_sha1_init,
 807                                 .cra_exit = img_hash_cra_exit,
 808                                 .cra_module = THIS_MODULE,
 809                         }
 810                 }
 811         },
 812         {
 813                 .init = img_hash_init,
 814                 .update = img_hash_update,
 815                 .final = img_hash_final,
 816                 .finup = img_hash_finup,
 817                 .export = img_hash_export,
 818                 .import = img_hash_import,
 819                 .digest = img_hash_digest,
 820                 .halg = {
 821                         .digestsize = SHA224_DIGEST_SIZE,
 822                         .statesize = sizeof(struct sha256_state),
 823                         .base = {
 824                                 .cra_name = "sha224",
 825                                 .cra_driver_name = "img-sha224",
 826                                 .cra_priority = 300,
 827                                 .cra_flags =
 828                                 CRYPTO_ALG_ASYNC |
 829                                 CRYPTO_ALG_NEED_FALLBACK,
 830                                 .cra_blocksize = SHA224_BLOCK_SIZE,
 831                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 832                                 .cra_init = img_hash_cra_sha224_init,
 833                                 .cra_exit = img_hash_cra_exit,
 834                                 .cra_module = THIS_MODULE,
 835                         }
 836                 }
 837         },
 838         {
 839                 .init = img_hash_init,
 840                 .update = img_hash_update,
 841                 .final = img_hash_final,
 842                 .finup = img_hash_finup,
 843                 .export = img_hash_export,
 844                 .import = img_hash_import,
 845                 .digest = img_hash_digest,
 846                 .halg = {
 847                         .digestsize = SHA256_DIGEST_SIZE,
 848                         .statesize = sizeof(struct sha256_state),
 849                         .base = {
 850                                 .cra_name = "sha256",
 851                                 .cra_driver_name = "img-sha256",
 852                                 .cra_priority = 300,
 853                                 .cra_flags =
 854                                 CRYPTO_ALG_ASYNC |
 855                                 CRYPTO_ALG_NEED_FALLBACK,
 856                                 .cra_blocksize = SHA256_BLOCK_SIZE,
 857                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 858                                 .cra_init = img_hash_cra_sha256_init,
 859                                 .cra_exit = img_hash_cra_exit,
 860                                 .cra_module = THIS_MODULE,
 861                         }
 862                 }
 863         }
 864 };
 865
 866 static int img_register_algs(struct img_hash_dev *hdev)
 867 {
 868         int i, err;
 869
 870         for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
 871                 err = crypto_register_ahash(&img_algs[i]);
 872                 if (err)
 873                         goto err_reg;
 874         }
 875         return 0;
 876
 877 err_reg:
 878         for (; i--; )
 879                 crypto_unregister_ahash(&img_algs[i]);
 880
 881         return err;
 882 }
 883
 884 static int img_unregister_algs(struct img_hash_dev *hdev)
 885 {
 886         int i;
 887
 888         for (i = 0; i < ARRAY_SIZE(img_algs); i++)
 889                 crypto_unregister_ahash(&img_algs[i]);
 890         return 0;
 891 }
 892
 893 static void img_hash_done_task(unsigned long data)
 894 {
 895         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 896         int err = 0;
 897
 898         if (hdev->err == -EINVAL) {
 899                 err = hdev->err;
 900                 goto finish;
 901         }
 902
 903         if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
 904                 img_hash_handle_queue(hdev, NULL);
 905                 return;
 906         }
 907
 908         if (DRIVER_FLAGS_CPU & hdev->flags) {
 909                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 910                         hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
 911                         goto finish;
 912                 }
 913         } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
 914                 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
 915                         hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
 916                         img_hash_write_via_dma_stop(hdev);
 917                         if (hdev->err) {
 918                                 err = hdev->err;
 919                                 goto finish;
 920                         }
 921                 }
 922                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 923                         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
 924                                         DRIVER_FLAGS_OUTPUT_READY);
 925                         goto finish;
 926                 }
 927         }
 928         return;
 929
 930 finish:
 931         img_hash_finish_req(hdev->req, err);
 932 }
 933
 934 static const struct of_device_id img_hash_match[] = {
 935         { .compatible = "img,hash-accelerator" },
 936         {}
 937 };
 938 MODULE_DEVICE_TABLE(of, img_hash_match);
 939
 940 static int img_hash_probe(struct platform_device *pdev)
 941 {
 942         struct img_hash_dev *hdev;
 943         struct device *dev = &pdev->dev;
 944         struct resource *hash_res;
 945         int     irq;
 946         int err;
 947
 948         hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
 949         if (hdev == NULL)
 950                 return -ENOMEM;
 951
 952         spin_lock_init(&hdev->lock);
 953
 954         hdev->dev = dev;
 955
 956         platform_set_drvdata(pdev, hdev);
 957
 958         INIT_LIST_HEAD(&hdev->list);
 959
 960         tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
 961         tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
 962
 963         crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
 964
 965         /* Register bank */
 966         hdev->io_base = devm_platform_ioremap_resource(pdev, 0);
 967         if (IS_ERR(hdev->io_base)) {
 968                 err = PTR_ERR(hdev->io_base);
 969                 dev_err(dev, "can't ioremap, returned %d\n", err);
 970
 971                 goto res_err;
 972         }
 973
 974         /* Write port (DMA or CPU) */
 975         hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 976         hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
 977         if (IS_ERR(hdev->cpu_addr)) {
 978                 dev_err(dev, "can't ioremap write port\n");
 979                 err = PTR_ERR(hdev->cpu_addr);
 980                 goto res_err;
 981         }
 982         hdev->bus_addr = hash_res->start;
 983
 984         irq = platform_get_irq(pdev, 0);
 985         if (irq < 0) {
 986                 err = irq;
 987                 goto res_err;
 988         }
 989
 990         err = devm_request_irq(dev, irq, img_irq_handler, 0,
 991                                dev_name(dev), hdev);
 992         if (err) {
 993                 dev_err(dev, "unable to request irq\n");
 994                 goto res_err;
 995         }
 996         dev_dbg(dev, "using IRQ channel %d\n", irq);
 997
 998         hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
 999         if (IS_ERR(hdev->hash_clk)) {
1000                 dev_err(dev, "clock initialization failed.\n");
1001                 err = PTR_ERR(hdev->hash_clk);
1002                 goto res_err;
1003         }
1004
1005         hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
1006         if (IS_ERR(hdev->sys_clk)) {
1007                 dev_err(dev, "clock initialization failed.\n");
1008                 err = PTR_ERR(hdev->sys_clk);
1009                 goto res_err;
1010         }
1011
1012         err = clk_prepare_enable(hdev->hash_clk);
1013         if (err)
1014                 goto res_err;
1015
1016         err = clk_prepare_enable(hdev->sys_clk);
1017         if (err)
1018                 goto clk_err;
1019
1020         err = img_hash_dma_init(hdev);
1021         if (err)
1022                 goto dma_err;
1023
1024         dev_dbg(dev, "using %s for DMA transfers\n",
1025                 dma_chan_name(hdev->dma_lch));
1026
1027         spin_lock(&img_hash.lock);
1028         list_add_tail(&hdev->list, &img_hash.dev_list);
1029         spin_unlock(&img_hash.lock);
1030
1031         err = img_register_algs(hdev);
1032         if (err)
1033                 goto err_algs;
1034         dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
1035
1036         return 0;
1037
1038 err_algs:
1039         spin_lock(&img_hash.lock);
1040         list_del(&hdev->list);
1041         spin_unlock(&img_hash.lock);
1042         dma_release_channel(hdev->dma_lch);
1043 dma_err:
1044         clk_disable_unprepare(hdev->sys_clk);
1045 clk_err:
1046         clk_disable_unprepare(hdev->hash_clk);
1047 res_err:
1048         tasklet_kill(&hdev->done_task);
1049         tasklet_kill(&hdev->dma_task);
1050
1051         return err;
1052 }
1053
1054 static int img_hash_remove(struct platform_device *pdev)
1055 {
1056         struct img_hash_dev *hdev;
1057
1058         hdev = platform_get_drvdata(pdev);
1059         spin_lock(&img_hash.lock);
1060         list_del(&hdev->list);
1061         spin_unlock(&img_hash.lock);
1062
1063         img_unregister_algs(hdev);
1064
1065         tasklet_kill(&hdev->done_task);
1066         tasklet_kill(&hdev->dma_task);
1067
1068         dma_release_channel(hdev->dma_lch);
1069
1070         clk_disable_unprepare(hdev->hash_clk);
1071         clk_disable_unprepare(hdev->sys_clk);
1072
1073         return 0;
1074 }
1075
1076 #ifdef CONFIG_PM_SLEEP
1077 static int img_hash_suspend(struct device *dev)
1078 {
1079         struct img_hash_dev *hdev = dev_get_drvdata(dev);
1080
1081         clk_disable_unprepare(hdev->hash_clk);
1082         clk_disable_unprepare(hdev->sys_clk);
1083
1084         return 0;
1085 }
1086
1087 static int img_hash_resume(struct device *dev)
1088 {
1089         struct img_hash_dev *hdev = dev_get_drvdata(dev);
1090         int ret;
1091
1092         ret = clk_prepare_enable(hdev->hash_clk);
1093         if (ret)
1094                 return ret;
1095
1096         ret = clk_prepare_enable(hdev->sys_clk);
1097         if (ret) {
1098                 clk_disable_unprepare(hdev->hash_clk);
1099                 return ret;
1100         }
1101
1102         return 0;
1103 }
1104 #endif /* CONFIG_PM_SLEEP */
1105
1106 static const struct dev_pm_ops img_hash_pm_ops = {
1107         SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)
1108 };
1109
1110 static struct platform_driver img_hash_driver = {
1111         .probe          = img_hash_probe,
1112         .remove         = img_hash_remove,
1113         .driver         = {
1114                 .name   = "img-hash-accelerator",
1115                 .pm     = &img_hash_pm_ops,
1116                 .of_match_table = of_match_ptr(img_hash_match),
1117         }
1118 };
1119 module_platform_driver(img_hash_driver);
1120
1121 MODULE_LICENSE("GPL v2");
1122 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1123 MODULE_AUTHOR("Will Thomas.");
1124 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");