1 // SPDX-License-Identifier: GPL-2.0-only
5 * Support for SAHARA cryptographic accelerator.
7 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
8 * Copyright (c) 2013 Vista Silicon S.L.
9 * Author: Javier Martin <javier.martin@vista-silicon.com>
11 * Based on omap-aes.c and tegra-aes.c
14 #include <crypto/aes.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/skcipher.h>
17 #include <crypto/scatterwalk.h>
18 #include <crypto/sha.h>
20 #include <linux/clk.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/interrupt.h>
24 #include <linux/irq.h>
25 #include <linux/kernel.h>
26 #include <linux/kthread.h>
27 #include <linux/module.h>
29 #include <linux/of_device.h>
30 #include <linux/platform_device.h>
31 #include <linux/spinlock.h>
33 #define SHA_BUFFER_LEN PAGE_SIZE
34 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
36 #define SAHARA_NAME "sahara"
37 #define SAHARA_VERSION_3 3
38 #define SAHARA_VERSION_4 4
39 #define SAHARA_TIMEOUT_MS 1000
40 #define SAHARA_MAX_HW_DESC 2
41 #define SAHARA_MAX_HW_LINK 20
43 #define FLAGS_MODE_MASK 0x000f
44 #define FLAGS_ENCRYPT BIT(0)
45 #define FLAGS_CBC BIT(1)
47 #define SAHARA_HDR_BASE 0x00800000
48 #define SAHARA_HDR_SKHA_ALG_AES 0
49 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
50 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
51 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
52 #define SAHARA_HDR_FORM_DATA (5 << 16)
53 #define SAHARA_HDR_FORM_KEY (8 << 16)
54 #define SAHARA_HDR_LLO (1 << 24)
55 #define SAHARA_HDR_CHA_SKHA (1 << 28)
56 #define SAHARA_HDR_CHA_MDHA (2 << 28)
57 #define SAHARA_HDR_PARITY_BIT (1 << 31)
59 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
60 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
61 #define SAHARA_HDR_MDHA_HASH 0xA0850000
62 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
63 #define SAHARA_HDR_MDHA_ALG_SHA1 0
64 #define SAHARA_HDR_MDHA_ALG_MD5 1
65 #define SAHARA_HDR_MDHA_ALG_SHA256 2
66 #define SAHARA_HDR_MDHA_ALG_SHA224 3
67 #define SAHARA_HDR_MDHA_PDATA (1 << 2)
68 #define SAHARA_HDR_MDHA_HMAC (1 << 3)
69 #define SAHARA_HDR_MDHA_INIT (1 << 5)
70 #define SAHARA_HDR_MDHA_IPAD (1 << 6)
71 #define SAHARA_HDR_MDHA_OPAD (1 << 7)
72 #define SAHARA_HDR_MDHA_SWAP (1 << 8)
73 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
74 #define SAHARA_HDR_MDHA_SSL (1 << 10)
76 /* SAHARA can only process one request at a time */
77 #define SAHARA_QUEUE_LENGTH 1
79 #define SAHARA_REG_VERSION 0x00
80 #define SAHARA_REG_DAR 0x04
81 #define SAHARA_REG_CONTROL 0x08
82 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
83 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
84 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
85 #define SAHARA_CONTROL_ENABLE_INT (1 << 4)
86 #define SAHARA_REG_CMD 0x0C
87 #define SAHARA_CMD_RESET (1 << 0)
88 #define SAHARA_CMD_CLEAR_INT (1 << 8)
89 #define SAHARA_CMD_CLEAR_ERR (1 << 9)
90 #define SAHARA_CMD_SINGLE_STEP (1 << 10)
91 #define SAHARA_CMD_MODE_BATCH (1 << 16)
92 #define SAHARA_CMD_MODE_DEBUG (1 << 18)
93 #define SAHARA_REG_STATUS 0x10
94 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
95 #define SAHARA_STATE_IDLE 0
96 #define SAHARA_STATE_BUSY 1
97 #define SAHARA_STATE_ERR 2
98 #define SAHARA_STATE_FAULT 3
99 #define SAHARA_STATE_COMPLETE 4
100 #define SAHARA_STATE_COMP_FLAG (1 << 2)
101 #define SAHARA_STATUS_DAR_FULL (1 << 3)
102 #define SAHARA_STATUS_ERROR (1 << 4)
103 #define SAHARA_STATUS_SECURE (1 << 5)
104 #define SAHARA_STATUS_FAIL (1 << 6)
105 #define SAHARA_STATUS_INIT (1 << 7)
106 #define SAHARA_STATUS_RNG_RESEED (1 << 8)
107 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
108 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
109 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
110 #define SAHARA_STATUS_MODE_BATCH (1 << 16)
111 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
112 #define SAHARA_STATUS_MODE_DEBUG (1 << 18)
113 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
114 #define SAHARA_REG_ERRSTATUS 0x14
115 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
116 #define SAHARA_ERRSOURCE_CHA 14
117 #define SAHARA_ERRSOURCE_DMA 15
118 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
119 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
120 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
121 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
122 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
123 #define SAHARA_REG_FADDR 0x18
124 #define SAHARA_REG_CDAR 0x1C
125 #define SAHARA_REG_IDAR 0x20
127 struct sahara_hw_desc {
136 struct sahara_hw_link {
143 /* AES-specific context */
145 u8 key[AES_KEYSIZE_128];
146 struct crypto_skcipher *fallback;
149 struct sahara_aes_reqctx {
151 u8 iv_out[AES_BLOCK_SIZE];
152 struct skcipher_request fallback_req; // keep at the end
156 * struct sahara_sha_reqctx - private data per request
157 * @buf: holds data for requests smaller than block_size
158 * @rembuf: used to prepare one block_size-aligned request
159 * @context: hw-specific context for request. Digest is extracted from this
160 * @mode: specifies what type of hw-descriptor needs to be built
161 * @digest_size: length of digest for this request
162 * @context_size: length of hw-context for this request.
163 * Always digest_size + 4
164 * @buf_cnt: number of bytes saved in buf
165 * @sg_in_idx: number of hw links
166 * @in_sg: scatterlist for input data
167 * @in_sg_chain: scatterlists for chained input data
168 * @total: total number of bytes for transfer
169 * @last: is this the last block
170 * @first: is this the first block
171 * @active: inside a transfer
173 struct sahara_sha_reqctx {
174 u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE];
175 u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
176 u8 context[SHA256_DIGEST_SIZE + 4];
178 unsigned int digest_size;
179 unsigned int context_size;
180 unsigned int buf_cnt;
181 unsigned int sg_in_idx;
182 struct scatterlist *in_sg;
183 struct scatterlist in_sg_chain[2];
191 struct device *device;
192 unsigned int version;
193 void __iomem *regs_base;
196 spinlock_t queue_spinlock;
197 struct task_struct *kthread;
198 struct completion dma_completion;
200 struct sahara_ctx *ctx;
201 struct crypto_queue queue;
204 struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC];
205 dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC];
208 dma_addr_t key_phys_base;
211 dma_addr_t iv_phys_base;
214 dma_addr_t context_phys_base;
216 struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK];
217 dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK];
220 struct scatterlist *in_sg;
222 struct scatterlist *out_sg;
228 static struct sahara_dev *dev_ptr;
230 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
232 writel(data, dev->regs_base + reg);
235 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
237 return readl(dev->regs_base + reg);
240 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
242 u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
243 SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
244 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
246 if (dev->flags & FLAGS_CBC) {
247 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
248 hdr ^= SAHARA_HDR_PARITY_BIT;
251 if (dev->flags & FLAGS_ENCRYPT) {
252 hdr |= SAHARA_HDR_SKHA_OP_ENC;
253 hdr ^= SAHARA_HDR_PARITY_BIT;
259 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
261 return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
262 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
265 static const char *sahara_err_src[16] = {
268 "Descriptor length error",
269 "Descriptor length or pointer error",
271 "Link pointer error",
272 "Input buffer error",
273 "Output buffer error",
274 "Output buffer starvation",
275 "Internal state fault",
276 "General descriptor problem",
278 "Descriptor address error",
279 "Link address error",
284 static const char *sahara_err_dmasize[4] = {
286 "Half-word transfer",
291 static const char *sahara_err_dmasrc[8] = {
294 "Internal IP bus error",
296 "DMA crosses 256 byte boundary",
302 static const char *sahara_cha_errsrc[12] = {
303 "Input buffer non-empty",
308 "Write during processing",
309 "CTX read during processing",
311 "Input buffer disabled/underflow",
312 "Output buffer disabled/overflow",
313 "DES key parity error",
317 static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
319 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
321 u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
322 u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
324 dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
326 dev_err(dev->device, " - %s.\n", sahara_err_src[source]);
328 if (source == SAHARA_ERRSOURCE_DMA) {
329 if (error & SAHARA_ERRSTATUS_DMA_DIR)
330 dev_err(dev->device, " * DMA read.\n");
332 dev_err(dev->device, " * DMA write.\n");
334 dev_err(dev->device, " * %s.\n",
335 sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
336 dev_err(dev->device, " * %s.\n",
337 sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
338 } else if (source == SAHARA_ERRSOURCE_CHA) {
339 dev_err(dev->device, " * %s.\n",
340 sahara_cha_errsrc[chasrc]);
341 dev_err(dev->device, " * %s.\n",
342 sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
344 dev_err(dev->device, "\n");
347 static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
349 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
353 if (!__is_defined(DEBUG))
356 state = SAHARA_STATUS_GET_STATE(status);
358 dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
361 dev_dbg(dev->device, " - State = %d:\n", state);
362 if (state & SAHARA_STATE_COMP_FLAG)
363 dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n");
365 dev_dbg(dev->device, " * %s.\n",
366 sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
368 if (status & SAHARA_STATUS_DAR_FULL)
369 dev_dbg(dev->device, " - DAR Full.\n");
370 if (status & SAHARA_STATUS_ERROR)
371 dev_dbg(dev->device, " - Error.\n");
372 if (status & SAHARA_STATUS_SECURE)
373 dev_dbg(dev->device, " - Secure.\n");
374 if (status & SAHARA_STATUS_FAIL)
375 dev_dbg(dev->device, " - Fail.\n");
376 if (status & SAHARA_STATUS_RNG_RESEED)
377 dev_dbg(dev->device, " - RNG Reseed Request.\n");
378 if (status & SAHARA_STATUS_ACTIVE_RNG)
379 dev_dbg(dev->device, " - RNG Active.\n");
380 if (status & SAHARA_STATUS_ACTIVE_MDHA)
381 dev_dbg(dev->device, " - MDHA Active.\n");
382 if (status & SAHARA_STATUS_ACTIVE_SKHA)
383 dev_dbg(dev->device, " - SKHA Active.\n");
385 if (status & SAHARA_STATUS_MODE_BATCH)
386 dev_dbg(dev->device, " - Batch Mode.\n");
387 else if (status & SAHARA_STATUS_MODE_DEDICATED)
388 dev_dbg(dev->device, " - Dedicated Mode.\n");
389 else if (status & SAHARA_STATUS_MODE_DEBUG)
390 dev_dbg(dev->device, " - Debug Mode.\n");
392 dev_dbg(dev->device, " - Internal state = 0x%02x\n",
393 SAHARA_STATUS_GET_ISTATE(status));
395 dev_dbg(dev->device, "Current DAR: 0x%08x\n",
396 sahara_read(dev, SAHARA_REG_CDAR));
397 dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
398 sahara_read(dev, SAHARA_REG_IDAR));
401 static void sahara_dump_descriptors(struct sahara_dev *dev)
405 if (!__is_defined(DEBUG))
408 for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
409 dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",
410 i, &dev->hw_phys_desc[i]);
411 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
412 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
413 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
414 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
415 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
416 dev_dbg(dev->device, "\tnext = 0x%08x\n",
417 dev->hw_desc[i]->next);
419 dev_dbg(dev->device, "\n");
422 static void sahara_dump_links(struct sahara_dev *dev)
426 if (!__is_defined(DEBUG))
429 for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
430 dev_dbg(dev->device, "Link (%d) (%pad):\n",
431 i, &dev->hw_phys_link[i]);
432 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
433 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
434 dev_dbg(dev->device, "\tnext = 0x%08x\n",
435 dev->hw_link[i]->next);
437 dev_dbg(dev->device, "\n");
440 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
442 struct sahara_ctx *ctx = dev->ctx;
443 struct scatterlist *sg;
449 memcpy(dev->key_base, ctx->key, ctx->keylen);
451 if (dev->flags & FLAGS_CBC) {
452 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
453 dev->hw_desc[idx]->p1 = dev->iv_phys_base;
455 dev->hw_desc[idx]->len1 = 0;
456 dev->hw_desc[idx]->p1 = 0;
458 dev->hw_desc[idx]->len2 = ctx->keylen;
459 dev->hw_desc[idx]->p2 = dev->key_phys_base;
460 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
461 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
466 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
467 if (dev->nb_in_sg < 0) {
468 dev_err(dev->device, "Invalid numbers of src SG.\n");
469 return dev->nb_in_sg;
471 dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
472 if (dev->nb_out_sg < 0) {
473 dev_err(dev->device, "Invalid numbers of dst SG.\n");
474 return dev->nb_out_sg;
476 if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
477 dev_err(dev->device, "not enough hw links (%d)\n",
478 dev->nb_in_sg + dev->nb_out_sg);
482 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
484 if (ret != dev->nb_in_sg) {
485 dev_err(dev->device, "couldn't map in sg\n");
489 ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
491 if (ret != dev->nb_out_sg) {
492 dev_err(dev->device, "couldn't map out sg\n");
496 /* Create input links */
497 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
500 for (i = 0; i < dev->nb_in_sg; i++) {
501 dev->hw_link[i]->len = min(len, sg->length);
502 dev->hw_link[i]->p = sg->dma_address;
503 if (i == (dev->nb_in_sg - 1)) {
504 dev->hw_link[i]->next = 0;
506 len -= min(len, sg->length);
507 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
512 /* Create output links */
513 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
516 for (j = i; j < dev->nb_out_sg + i; j++) {
517 dev->hw_link[j]->len = min(len, sg->length);
518 dev->hw_link[j]->p = sg->dma_address;
519 if (j == (dev->nb_out_sg + i - 1)) {
520 dev->hw_link[j]->next = 0;
522 len -= min(len, sg->length);
523 dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
528 /* Fill remaining fields of hw_desc[1] */
529 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
530 dev->hw_desc[idx]->len1 = dev->total;
531 dev->hw_desc[idx]->len2 = dev->total;
532 dev->hw_desc[idx]->next = 0;
534 sahara_dump_descriptors(dev);
535 sahara_dump_links(dev);
537 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
542 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
548 static void sahara_aes_cbc_update_iv(struct skcipher_request *req)
550 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
551 struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
552 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
554 /* Update IV buffer to contain the last ciphertext block */
555 if (rctx->mode & FLAGS_ENCRYPT) {
556 sg_pcopy_to_buffer(req->dst, sg_nents(req->dst), req->iv,
557 ivsize, req->cryptlen - ivsize);
559 memcpy(req->iv, rctx->iv_out, ivsize);
563 static int sahara_aes_process(struct skcipher_request *req)
565 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
566 struct sahara_dev *dev = dev_ptr;
567 struct sahara_ctx *ctx;
568 struct sahara_aes_reqctx *rctx;
570 unsigned long timeout;
572 /* Request is ready to be dispatched by the device */
574 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
575 req->cryptlen, req->src, req->dst);
577 /* assign new request to device */
578 dev->total = req->cryptlen;
579 dev->in_sg = req->src;
580 dev->out_sg = req->dst;
582 rctx = skcipher_request_ctx(req);
583 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
584 rctx->mode &= FLAGS_MODE_MASK;
585 dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
587 if ((dev->flags & FLAGS_CBC) && req->iv) {
588 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
590 memcpy(dev->iv_base, req->iv, ivsize);
592 if (!(dev->flags & FLAGS_ENCRYPT)) {
593 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
594 rctx->iv_out, ivsize,
595 req->cryptlen - ivsize);
599 /* assign new context to device */
602 reinit_completion(&dev->dma_completion);
604 ret = sahara_hw_descriptor_create(dev);
608 timeout = wait_for_completion_timeout(&dev->dma_completion,
609 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
611 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
613 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
617 dev_err(dev->device, "AES timeout\n");
621 if ((dev->flags & FLAGS_CBC) && req->iv)
622 sahara_aes_cbc_update_iv(req);
627 static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
630 struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
632 ctx->keylen = keylen;
634 /* SAHARA only supports 128bit keys */
635 if (keylen == AES_KEYSIZE_128) {
636 memcpy(ctx->key, key, keylen);
640 if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
644 * The requested key size is not supported by HW, do a fallback.
646 crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
647 crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
648 CRYPTO_TFM_REQ_MASK);
649 return crypto_skcipher_setkey(ctx->fallback, key, keylen);
652 static int sahara_aes_fallback(struct skcipher_request *req, unsigned long mode)
654 struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
655 struct sahara_ctx *ctx = crypto_skcipher_ctx(
656 crypto_skcipher_reqtfm(req));
658 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
659 skcipher_request_set_callback(&rctx->fallback_req,
663 skcipher_request_set_crypt(&rctx->fallback_req, req->src,
664 req->dst, req->cryptlen, req->iv);
666 if (mode & FLAGS_ENCRYPT)
667 return crypto_skcipher_encrypt(&rctx->fallback_req);
669 return crypto_skcipher_decrypt(&rctx->fallback_req);
672 static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode)
674 struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
675 struct sahara_ctx *ctx = crypto_skcipher_ctx(
676 crypto_skcipher_reqtfm(req));
677 struct sahara_dev *dev = dev_ptr;
683 if (unlikely(ctx->keylen != AES_KEYSIZE_128))
684 return sahara_aes_fallback(req, mode);
686 dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
687 req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
689 if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) {
691 "request size is not exact amount of AES blocks\n");
697 spin_lock_bh(&dev->queue_spinlock);
698 err = crypto_enqueue_request(&dev->queue, &req->base);
699 spin_unlock_bh(&dev->queue_spinlock);
701 wake_up_process(dev->kthread);
706 static int sahara_aes_ecb_encrypt(struct skcipher_request *req)
708 return sahara_aes_crypt(req, FLAGS_ENCRYPT);
711 static int sahara_aes_ecb_decrypt(struct skcipher_request *req)
713 return sahara_aes_crypt(req, 0);
716 static int sahara_aes_cbc_encrypt(struct skcipher_request *req)
718 return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
721 static int sahara_aes_cbc_decrypt(struct skcipher_request *req)
723 return sahara_aes_crypt(req, FLAGS_CBC);
726 static int sahara_aes_init_tfm(struct crypto_skcipher *tfm)
728 const char *name = crypto_tfm_alg_name(&tfm->base);
729 struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
731 ctx->fallback = crypto_alloc_skcipher(name, 0,
732 CRYPTO_ALG_NEED_FALLBACK);
733 if (IS_ERR(ctx->fallback)) {
734 pr_err("Error allocating fallback algo %s\n", name);
735 return PTR_ERR(ctx->fallback);
738 crypto_skcipher_set_reqsize(tfm, sizeof(struct sahara_aes_reqctx) +
739 crypto_skcipher_reqsize(ctx->fallback));
744 static void sahara_aes_exit_tfm(struct crypto_skcipher *tfm)
746 struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
748 crypto_free_skcipher(ctx->fallback);
751 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
752 struct sahara_sha_reqctx *rctx)
759 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
760 hdr |= SAHARA_HDR_MDHA_INIT;
762 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
766 hdr |= SAHARA_HDR_MDHA_PDATA;
768 if (hweight_long(hdr) % 2 == 0)
769 hdr |= SAHARA_HDR_PARITY_BIT;
774 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
775 struct sahara_sha_reqctx *rctx,
778 struct scatterlist *sg;
783 dev->in_sg = rctx->in_sg;
785 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
786 if (dev->nb_in_sg < 0) {
787 dev_err(dev->device, "Invalid numbers of src SG.\n");
788 return dev->nb_in_sg;
790 if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
791 dev_err(dev->device, "not enough hw links (%d)\n",
792 dev->nb_in_sg + dev->nb_out_sg);
797 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
802 for (i = start; i < dev->nb_in_sg + start; i++) {
803 dev->hw_link[i]->len = min(len, sg->length);
804 dev->hw_link[i]->p = sg->dma_address;
805 if (i == (dev->nb_in_sg + start - 1)) {
806 dev->hw_link[i]->next = 0;
808 len -= min(len, sg->length);
809 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
817 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
818 struct sahara_sha_reqctx *rctx,
819 struct ahash_request *req,
826 /* Create initial descriptor: #8*/
827 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
829 /* Create hash descriptor: #10. Must follow #6. */
830 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
832 dev->hw_desc[index]->len1 = rctx->total;
833 if (dev->hw_desc[index]->len1 == 0) {
834 /* if len1 is 0, p1 must be 0, too */
835 dev->hw_desc[index]->p1 = 0;
838 /* Create input links */
839 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
840 i = sahara_sha_hw_links_create(dev, rctx, index);
842 rctx->sg_in_idx = index;
847 dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
849 /* Save the context for the next operation */
850 result_len = rctx->context_size;
851 dev->hw_link[i]->p = dev->context_phys_base;
853 dev->hw_link[i]->len = result_len;
854 dev->hw_desc[index]->len2 = result_len;
856 dev->hw_link[i]->next = 0;
862 * Load descriptor aka #6
864 * To load a previously saved context back to the MDHA unit
870 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
871 struct sahara_sha_reqctx *rctx,
872 struct ahash_request *req,
875 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
877 dev->hw_desc[index]->len1 = rctx->context_size;
878 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
879 dev->hw_desc[index]->len2 = 0;
880 dev->hw_desc[index]->p2 = 0;
882 dev->hw_link[index]->len = rctx->context_size;
883 dev->hw_link[index]->p = dev->context_phys_base;
884 dev->hw_link[index]->next = 0;
889 static int sahara_sha_prepare_request(struct ahash_request *req)
891 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
892 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
893 unsigned int hash_later;
894 unsigned int block_size;
897 block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
899 /* append bytes from previous operation */
900 len = rctx->buf_cnt + req->nbytes;
902 /* only the last transfer can be padded in hardware */
903 if (!rctx->last && (len < block_size)) {
904 /* to few data, save for next operation */
905 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
907 rctx->buf_cnt += req->nbytes;
912 /* add data from previous operation first */
914 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
916 /* data must always be a multiple of block_size */
917 hash_later = rctx->last ? 0 : len & (block_size - 1);
919 unsigned int offset = req->nbytes - hash_later;
920 /* Save remaining bytes for later use */
921 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
925 rctx->total = len - hash_later;
926 /* have data from previous operation and current */
927 if (rctx->buf_cnt && req->nbytes) {
928 sg_init_table(rctx->in_sg_chain, 2);
929 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
930 sg_chain(rctx->in_sg_chain, 2, req->src);
931 rctx->in_sg = rctx->in_sg_chain;
932 /* only data from previous operation */
933 } else if (rctx->buf_cnt) {
934 rctx->in_sg = rctx->in_sg_chain;
935 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
936 /* no data from previous operation */
938 rctx->in_sg = req->src;
941 /* on next call, we only have the remaining data in the buffer */
942 rctx->buf_cnt = hash_later;
947 static int sahara_sha_process(struct ahash_request *req)
949 struct sahara_dev *dev = dev_ptr;
950 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
952 unsigned long timeout;
954 ret = sahara_sha_prepare_request(req);
959 ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
963 dev->hw_desc[0]->next = 0;
966 memcpy(dev->context_base, rctx->context, rctx->context_size);
968 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
969 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
970 ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
974 dev->hw_desc[1]->next = 0;
977 sahara_dump_descriptors(dev);
978 sahara_dump_links(dev);
980 reinit_completion(&dev->dma_completion);
982 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
984 timeout = wait_for_completion_timeout(&dev->dma_completion,
985 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
988 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
992 dev_err(dev->device, "SHA timeout\n");
996 memcpy(rctx->context, dev->context_base, rctx->context_size);
998 if (req->result && rctx->last)
999 memcpy(req->result, rctx->context, rctx->digest_size);
1004 static int sahara_queue_manage(void *data)
1006 struct sahara_dev *dev = (struct sahara_dev *)data;
1007 struct crypto_async_request *async_req;
1008 struct crypto_async_request *backlog;
1012 __set_current_state(TASK_INTERRUPTIBLE);
1014 spin_lock_bh(&dev->queue_spinlock);
1015 backlog = crypto_get_backlog(&dev->queue);
1016 async_req = crypto_dequeue_request(&dev->queue);
1017 spin_unlock_bh(&dev->queue_spinlock);
1020 backlog->complete(backlog, -EINPROGRESS);
1023 if (crypto_tfm_alg_type(async_req->tfm) ==
1024 CRYPTO_ALG_TYPE_AHASH) {
1025 struct ahash_request *req =
1026 ahash_request_cast(async_req);
1028 ret = sahara_sha_process(req);
1030 struct skcipher_request *req =
1031 skcipher_request_cast(async_req);
1033 ret = sahara_aes_process(req);
1036 async_req->complete(async_req, ret);
1042 } while (!kthread_should_stop());
1047 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1049 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1050 struct sahara_dev *dev = dev_ptr;
1053 if (!req->nbytes && !last)
1058 if (!rctx->active) {
1063 spin_lock_bh(&dev->queue_spinlock);
1064 ret = crypto_enqueue_request(&dev->queue, &req->base);
1065 spin_unlock_bh(&dev->queue_spinlock);
1067 wake_up_process(dev->kthread);
1072 static int sahara_sha_init(struct ahash_request *req)
1074 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1075 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1077 memset(rctx, 0, sizeof(*rctx));
1079 switch (crypto_ahash_digestsize(tfm)) {
1080 case SHA1_DIGEST_SIZE:
1081 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1082 rctx->digest_size = SHA1_DIGEST_SIZE;
1084 case SHA256_DIGEST_SIZE:
1085 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1086 rctx->digest_size = SHA256_DIGEST_SIZE;
1092 rctx->context_size = rctx->digest_size + 4;
1098 static int sahara_sha_update(struct ahash_request *req)
1100 return sahara_sha_enqueue(req, 0);
1103 static int sahara_sha_final(struct ahash_request *req)
1106 return sahara_sha_enqueue(req, 1);
1109 static int sahara_sha_finup(struct ahash_request *req)
1111 return sahara_sha_enqueue(req, 1);
1114 static int sahara_sha_digest(struct ahash_request *req)
1116 sahara_sha_init(req);
1118 return sahara_sha_finup(req);
1121 static int sahara_sha_export(struct ahash_request *req, void *out)
1123 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1125 memcpy(out, rctx, sizeof(struct sahara_sha_reqctx));
1130 static int sahara_sha_import(struct ahash_request *req, const void *in)
1132 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1134 memcpy(rctx, in, sizeof(struct sahara_sha_reqctx));
1139 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1141 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1142 sizeof(struct sahara_sha_reqctx));
1147 static struct skcipher_alg aes_algs[] = {
1149 .base.cra_name = "ecb(aes)",
1150 .base.cra_driver_name = "sahara-ecb-aes",
1151 .base.cra_priority = 300,
1152 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1153 .base.cra_blocksize = AES_BLOCK_SIZE,
1154 .base.cra_ctxsize = sizeof(struct sahara_ctx),
1155 .base.cra_alignmask = 0x0,
1156 .base.cra_module = THIS_MODULE,
1158 .init = sahara_aes_init_tfm,
1159 .exit = sahara_aes_exit_tfm,
1160 .min_keysize = AES_MIN_KEY_SIZE ,
1161 .max_keysize = AES_MAX_KEY_SIZE,
1162 .setkey = sahara_aes_setkey,
1163 .encrypt = sahara_aes_ecb_encrypt,
1164 .decrypt = sahara_aes_ecb_decrypt,
1166 .base.cra_name = "cbc(aes)",
1167 .base.cra_driver_name = "sahara-cbc-aes",
1168 .base.cra_priority = 300,
1169 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1170 .base.cra_blocksize = AES_BLOCK_SIZE,
1171 .base.cra_ctxsize = sizeof(struct sahara_ctx),
1172 .base.cra_alignmask = 0x0,
1173 .base.cra_module = THIS_MODULE,
1175 .init = sahara_aes_init_tfm,
1176 .exit = sahara_aes_exit_tfm,
1177 .min_keysize = AES_MIN_KEY_SIZE ,
1178 .max_keysize = AES_MAX_KEY_SIZE,
1179 .ivsize = AES_BLOCK_SIZE,
1180 .setkey = sahara_aes_setkey,
1181 .encrypt = sahara_aes_cbc_encrypt,
1182 .decrypt = sahara_aes_cbc_decrypt,
1186 static struct ahash_alg sha_v3_algs[] = {
1188 .init = sahara_sha_init,
1189 .update = sahara_sha_update,
1190 .final = sahara_sha_final,
1191 .finup = sahara_sha_finup,
1192 .digest = sahara_sha_digest,
1193 .export = sahara_sha_export,
1194 .import = sahara_sha_import,
1195 .halg.digestsize = SHA1_DIGEST_SIZE,
1196 .halg.statesize = sizeof(struct sahara_sha_reqctx),
1199 .cra_driver_name = "sahara-sha1",
1200 .cra_priority = 300,
1201 .cra_flags = CRYPTO_ALG_ASYNC |
1202 CRYPTO_ALG_NEED_FALLBACK,
1203 .cra_blocksize = SHA1_BLOCK_SIZE,
1204 .cra_ctxsize = sizeof(struct sahara_ctx),
1206 .cra_module = THIS_MODULE,
1207 .cra_init = sahara_sha_cra_init,
1212 static struct ahash_alg sha_v4_algs[] = {
1214 .init = sahara_sha_init,
1215 .update = sahara_sha_update,
1216 .final = sahara_sha_final,
1217 .finup = sahara_sha_finup,
1218 .digest = sahara_sha_digest,
1219 .export = sahara_sha_export,
1220 .import = sahara_sha_import,
1221 .halg.digestsize = SHA256_DIGEST_SIZE,
1222 .halg.statesize = sizeof(struct sahara_sha_reqctx),
1224 .cra_name = "sha256",
1225 .cra_driver_name = "sahara-sha256",
1226 .cra_priority = 300,
1227 .cra_flags = CRYPTO_ALG_ASYNC |
1228 CRYPTO_ALG_NEED_FALLBACK,
1229 .cra_blocksize = SHA256_BLOCK_SIZE,
1230 .cra_ctxsize = sizeof(struct sahara_ctx),
1232 .cra_module = THIS_MODULE,
1233 .cra_init = sahara_sha_cra_init,
1238 static irqreturn_t sahara_irq_handler(int irq, void *data)
1240 struct sahara_dev *dev = (struct sahara_dev *)data;
1241 unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1242 unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1244 sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1247 sahara_decode_status(dev, stat);
1249 if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1251 } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1254 sahara_decode_error(dev, err);
1255 dev->error = -EINVAL;
1258 complete(&dev->dma_completion);
1264 static int sahara_register_algs(struct sahara_dev *dev)
1267 unsigned int i, j, k, l;
1269 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1270 err = crypto_register_skcipher(&aes_algs[i]);
1275 for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1276 err = crypto_register_ahash(&sha_v3_algs[k]);
1278 goto err_sha_v3_algs;
1281 if (dev->version > SAHARA_VERSION_3)
1282 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1283 err = crypto_register_ahash(&sha_v4_algs[l]);
1285 goto err_sha_v4_algs;
1291 for (j = 0; j < l; j++)
1292 crypto_unregister_ahash(&sha_v4_algs[j]);
1295 for (j = 0; j < k; j++)
1296 crypto_unregister_ahash(&sha_v3_algs[j]);
1299 for (j = 0; j < i; j++)
1300 crypto_unregister_skcipher(&aes_algs[j]);
1305 static void sahara_unregister_algs(struct sahara_dev *dev)
1309 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1310 crypto_unregister_skcipher(&aes_algs[i]);
1312 for (i = 0; i < ARRAY_SIZE(sha_v3_algs); i++)
1313 crypto_unregister_ahash(&sha_v3_algs[i]);
1315 if (dev->version > SAHARA_VERSION_3)
1316 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1317 crypto_unregister_ahash(&sha_v4_algs[i]);
1320 static const struct platform_device_id sahara_platform_ids[] = {
1321 { .name = "sahara-imx27" },
1324 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1326 static const struct of_device_id sahara_dt_ids[] = {
1327 { .compatible = "fsl,imx53-sahara" },
1328 { .compatible = "fsl,imx27-sahara" },
1331 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1333 static int sahara_probe(struct platform_device *pdev)
1335 struct sahara_dev *dev;
1341 dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
1345 dev->device = &pdev->dev;
1346 platform_set_drvdata(pdev, dev);
1348 /* Get the base address */
1349 dev->regs_base = devm_platform_ioremap_resource(pdev, 0);
1350 if (IS_ERR(dev->regs_base))
1351 return PTR_ERR(dev->regs_base);
1354 irq = platform_get_irq(pdev, 0);
1358 err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1359 0, dev_name(&pdev->dev), dev);
1361 dev_err(&pdev->dev, "failed to request irq\n");
1366 dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1367 if (IS_ERR(dev->clk_ipg)) {
1368 dev_err(&pdev->dev, "Could not get ipg clock\n");
1369 return PTR_ERR(dev->clk_ipg);
1372 dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1373 if (IS_ERR(dev->clk_ahb)) {
1374 dev_err(&pdev->dev, "Could not get ahb clock\n");
1375 return PTR_ERR(dev->clk_ahb);
1378 /* Allocate HW descriptors */
1379 dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
1380 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1381 &dev->hw_phys_desc[0], GFP_KERNEL);
1382 if (!dev->hw_desc[0]) {
1383 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1386 dev->hw_desc[1] = dev->hw_desc[0] + 1;
1387 dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1388 sizeof(struct sahara_hw_desc);
1390 /* Allocate space for iv and key */
1391 dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1392 &dev->key_phys_base, GFP_KERNEL);
1393 if (!dev->key_base) {
1394 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1397 dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1398 dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1400 /* Allocate space for context: largest digest + message length field */
1401 dev->context_base = dmam_alloc_coherent(&pdev->dev,
1402 SHA256_DIGEST_SIZE + 4,
1403 &dev->context_phys_base, GFP_KERNEL);
1404 if (!dev->context_base) {
1405 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1409 /* Allocate space for HW links */
1410 dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
1411 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1412 &dev->hw_phys_link[0], GFP_KERNEL);
1413 if (!dev->hw_link[0]) {
1414 dev_err(&pdev->dev, "Could not allocate hw links\n");
1417 for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1418 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1419 sizeof(struct sahara_hw_link);
1420 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1423 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1425 spin_lock_init(&dev->queue_spinlock);
1429 dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1430 if (IS_ERR(dev->kthread)) {
1431 return PTR_ERR(dev->kthread);
1434 init_completion(&dev->dma_completion);
1436 err = clk_prepare_enable(dev->clk_ipg);
1439 err = clk_prepare_enable(dev->clk_ahb);
1441 goto clk_ipg_disable;
1443 version = sahara_read(dev, SAHARA_REG_VERSION);
1444 if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1445 if (version != SAHARA_VERSION_3)
1447 } else if (of_device_is_compatible(pdev->dev.of_node,
1448 "fsl,imx53-sahara")) {
1449 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1451 version = (version >> 8) & 0xff;
1453 if (err == -ENODEV) {
1454 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1459 dev->version = version;
1461 sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1463 sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1464 SAHARA_CONTROL_SET_MAXBURST(8) |
1465 SAHARA_CONTROL_RNG_AUTORSD |
1466 SAHARA_CONTROL_ENABLE_INT,
1467 SAHARA_REG_CONTROL);
1469 err = sahara_register_algs(dev);
1473 dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1478 kthread_stop(dev->kthread);
1480 clk_disable_unprepare(dev->clk_ahb);
1482 clk_disable_unprepare(dev->clk_ipg);
1487 static int sahara_remove(struct platform_device *pdev)
1489 struct sahara_dev *dev = platform_get_drvdata(pdev);
1491 kthread_stop(dev->kthread);
1493 sahara_unregister_algs(dev);
1495 clk_disable_unprepare(dev->clk_ipg);
1496 clk_disable_unprepare(dev->clk_ahb);
1503 static struct platform_driver sahara_driver = {
1504 .probe = sahara_probe,
1505 .remove = sahara_remove,
1507 .name = SAHARA_NAME,
1508 .of_match_table = sahara_dt_ids,
1510 .id_table = sahara_platform_ids,
1513 module_platform_driver(sahara_driver);
1515 MODULE_LICENSE("GPL");
1516 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1517 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1518 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");