1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) ST-Ericsson SA 2010
4 * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson.
5 * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson.
6 * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson.
7 * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson.
8 * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson.
9 * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson.
12 #include <linux/clk.h>
13 #include <linux/completion.h>
14 #include <linux/device.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmaengine.h>
17 #include <linux/err.h>
18 #include <linux/errno.h>
19 #include <linux/interrupt.h>
21 #include <linux/irqreturn.h>
22 #include <linux/kernel.h>
23 #include <linux/klist.h>
24 #include <linux/module.h>
25 #include <linux/mod_devicetable.h>
26 #include <linux/platform_device.h>
27 #include <linux/regulator/consumer.h>
28 #include <linux/semaphore.h>
29 #include <linux/platform_data/dma-ste-dma40.h>
31 #include <crypto/aes.h>
32 #include <crypto/ctr.h>
33 #include <crypto/internal/des.h>
34 #include <crypto/internal/skcipher.h>
35 #include <crypto/scatterwalk.h>
37 #include <linux/platform_data/crypto-ux500.h>
42 #define CRYP_MAX_KEY_SIZE 32
43 #define BYTES_PER_WORD 4
46 static atomic_t session_id;
48 static struct stedma40_chan_cfg *mem_to_engine;
49 static struct stedma40_chan_cfg *engine_to_mem;
52 * struct cryp_driver_data - data specific to the driver.
54 * @device_list: A list of registered devices to choose from.
55 * @device_allocation: A semaphore initialized with number of devices.
57 struct cryp_driver_data {
58 struct klist device_list;
59 struct semaphore device_allocation;
63 * struct cryp_ctx - Crypto context
64 * @config: Crypto mode.
66 * @keylen: Length of key.
67 * @iv: Pointer to initialization vector.
68 * @indata: Pointer to indata.
69 * @outdata: Pointer to outdata.
70 * @datalen: Length of indata.
71 * @outlen: Length of outdata.
72 * @blocksize: Size of blocks.
73 * @updated: Updated flag.
74 * @dev_ctx: Device dependent context.
75 * @device: Pointer to the device.
76 * @session_id: Atomic session ID.
79 struct cryp_config config;
80 u8 key[CRYP_MAX_KEY_SIZE];
89 struct cryp_device_context dev_ctx;
90 struct cryp_device_data *device;
94 static struct cryp_driver_data driver_data;
97 * swap_bits_in_byte - mirror the bits in a byte
98 * @b: the byte to be mirrored
100 * The bits are swapped the following way:
101 * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and
102 * nibble 2 (n2) bits 4-7.
105 * (The "old" (moved) bit is replaced with a zero)
106 * 1. Move bit 6 and 7, 4 positions to the left.
107 * 2. Move bit 3 and 5, 2 positions to the left.
108 * 3. Move bit 1-4, 1 position to the left.
111 * 1. Move bit 0 and 1, 4 positions to the right.
112 * 2. Move bit 2 and 4, 2 positions to the right.
113 * 3. Move bit 3-6, 1 position to the right.
115 * Combine the two nibbles to a complete and swapped byte.
118 static inline u8 swap_bits_in_byte(u8 b)
120 #define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */
121 #define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5,
123 #define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4,
125 #define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */
126 #define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4,
128 #define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6,
134 /* Swap most significant nibble */
135 /* Right shift 4, bits 6 and 7 */
136 n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4));
137 /* Right shift 2, bits 3 and 5 */
138 n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));
139 /* Right shift 1, bits 1-4 */
140 n1 = (n1 & R_SHIFT_1_MASK) >> 1;
142 /* Swap least significant nibble */
143 /* Left shift 4, bits 0 and 1 */
144 n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4));
145 /* Left shift 2, bits 2 and 4 */
146 n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));
147 /* Left shift 1, bits 3-6 */
148 n2 = (n2 & L_SHIFT_1_MASK) << 1;
153 static inline void swap_words_in_key_and_bits_in_byte(const u8 *in,
160 j = len - BYTES_PER_WORD;
162 for (i = 0; i < BYTES_PER_WORD; i++) {
163 index = len - j - BYTES_PER_WORD + i;
165 swap_bits_in_byte(in[index]);
171 static void add_session_id(struct cryp_ctx *ctx)
174 * We never want 0 to be a valid value, since this is the default value
175 * for the software context.
177 if (unlikely(atomic_inc_and_test(&session_id)))
178 atomic_inc(&session_id);
180 ctx->session_id = atomic_read(&session_id);
183 static irqreturn_t cryp_interrupt_handler(int irq, void *param)
185 struct cryp_ctx *ctx;
187 struct cryp_device_data *device_data;
194 /* The device is coming from the one found in hw_crypt_noxts. */
195 device_data = (struct cryp_device_data *)param;
197 ctx = device_data->current_ctx;
204 dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen,
205 cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ?
208 if (cryp_pending_irq_src(device_data,
209 CRYP_IRQ_SRC_OUTPUT_FIFO)) {
210 if (ctx->outlen / ctx->blocksize > 0) {
211 count = ctx->blocksize / 4;
213 readsl(&device_data->base->dout, ctx->outdata, count);
214 ctx->outdata += count;
215 ctx->outlen -= count;
217 if (ctx->outlen == 0) {
218 cryp_disable_irq_src(device_data,
219 CRYP_IRQ_SRC_OUTPUT_FIFO);
222 } else if (cryp_pending_irq_src(device_data,
223 CRYP_IRQ_SRC_INPUT_FIFO)) {
224 if (ctx->datalen / ctx->blocksize > 0) {
225 count = ctx->blocksize / 4;
227 writesl(&device_data->base->din, ctx->indata, count);
229 ctx->indata += count;
230 ctx->datalen -= count;
232 if (ctx->datalen == 0)
233 cryp_disable_irq_src(device_data,
234 CRYP_IRQ_SRC_INPUT_FIFO);
236 if (ctx->config.algomode == CRYP_ALGO_AES_XTS) {
237 CRYP_PUT_BITS(&device_data->base->cr,
242 cryp_wait_until_done(device_data);
250 static int mode_is_aes(enum cryp_algo_mode mode)
252 return CRYP_ALGO_AES_ECB == mode ||
253 CRYP_ALGO_AES_CBC == mode ||
254 CRYP_ALGO_AES_CTR == mode ||
255 CRYP_ALGO_AES_XTS == mode;
258 static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right,
259 enum cryp_init_vector_index index)
261 struct cryp_init_vector_value vector_value;
263 dev_dbg(device_data->dev, "[%s]", __func__);
265 vector_value.init_value_left = left;
266 vector_value.init_value_right = right;
268 return cryp_configure_init_vector(device_data,
273 static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx)
277 int num_of_regs = ctx->blocksize / 8;
278 __be32 *civ = (__be32 *)ctx->iv;
279 u32 iv[AES_BLOCK_SIZE / 4];
281 dev_dbg(device_data->dev, "[%s]", __func__);
284 * Since we loop on num_of_regs we need to have a check in case
285 * someone provides an incorrect blocksize which would force calling
286 * cfg_iv with i greater than 2 which is an error.
288 if (num_of_regs > 2) {
289 dev_err(device_data->dev, "[%s] Incorrect blocksize %d",
290 __func__, ctx->blocksize);
294 for (i = 0; i < ctx->blocksize / 4; i++)
295 iv[i] = be32_to_cpup(civ + i);
297 for (i = 0; i < num_of_regs; i++) {
298 status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
299 (enum cryp_init_vector_index) i);
306 static int set_key(struct cryp_device_data *device_data,
309 enum cryp_key_reg_index index)
311 struct cryp_key_value key_value;
314 dev_dbg(device_data->dev, "[%s]", __func__);
316 key_value.key_value_left = left_key;
317 key_value.key_value_right = right_key;
319 cryp_error = cryp_configure_key_values(device_data,
323 dev_err(device_data->dev, "[%s]: "
324 "cryp_configure_key_values() failed!", __func__);
329 static int cfg_keys(struct cryp_ctx *ctx)
332 int num_of_regs = ctx->keylen / 8;
333 u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
334 __be32 *ckey = (__be32 *)ctx->key;
337 dev_dbg(ctx->device->dev, "[%s]", __func__);
339 if (mode_is_aes(ctx->config.algomode)) {
340 swap_words_in_key_and_bits_in_byte((u8 *)ckey,
344 for (i = 0; i < ctx->keylen / 4; i++)
345 swapped_key[i] = be32_to_cpup(ckey + i);
348 for (i = 0; i < num_of_regs; i++) {
349 cryp_error = set_key(ctx->device,
351 swapped_key[i * 2 + 1],
352 (enum cryp_key_reg_index) i);
354 if (cryp_error != 0) {
355 dev_err(ctx->device->dev, "[%s]: set_key() failed!",
363 static int cryp_setup_context(struct cryp_ctx *ctx,
364 struct cryp_device_data *device_data)
366 u32 control_register = CRYP_CR_DEFAULT;
369 case CRYP_MODE_INTERRUPT:
370 writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc);
374 writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr);
381 if (ctx->updated == 0) {
382 cryp_flush_inoutfifo(device_data);
383 if (cfg_keys(ctx) != 0) {
384 dev_err(ctx->device->dev, "[%s]: cfg_keys failed!",
390 CRYP_ALGO_AES_ECB != ctx->config.algomode &&
391 CRYP_ALGO_DES_ECB != ctx->config.algomode &&
392 CRYP_ALGO_TDES_ECB != ctx->config.algomode) {
393 if (cfg_ivs(device_data, ctx) != 0)
397 cryp_set_configuration(device_data, &ctx->config,
400 } else if (ctx->updated == 1 &&
401 ctx->session_id != atomic_read(&session_id)) {
402 cryp_flush_inoutfifo(device_data);
403 cryp_restore_device_context(device_data, &ctx->dev_ctx);
406 control_register = ctx->dev_ctx.cr;
408 control_register = ctx->dev_ctx.cr;
410 writel(control_register |
411 (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS),
412 &device_data->base->cr);
417 static int cryp_get_device_data(struct cryp_ctx *ctx,
418 struct cryp_device_data **device_data)
421 struct klist_iter device_iterator;
422 struct klist_node *device_node;
423 struct cryp_device_data *local_device_data = NULL;
424 pr_debug(DEV_DBG_NAME " [%s]", __func__);
426 /* Wait until a device is available */
427 ret = down_interruptible(&driver_data.device_allocation);
429 return ret; /* Interrupted */
431 /* Select a device */
432 klist_iter_init(&driver_data.device_list, &device_iterator);
434 device_node = klist_next(&device_iterator);
435 while (device_node) {
436 local_device_data = container_of(device_node,
437 struct cryp_device_data, list_node);
438 spin_lock(&local_device_data->ctx_lock);
439 /* current_ctx allocates a device, NULL = unallocated */
440 if (local_device_data->current_ctx) {
441 device_node = klist_next(&device_iterator);
443 local_device_data->current_ctx = ctx;
444 ctx->device = local_device_data;
445 spin_unlock(&local_device_data->ctx_lock);
448 spin_unlock(&local_device_data->ctx_lock);
450 klist_iter_exit(&device_iterator);
454 * No free device found.
455 * Since we allocated a device with down_interruptible, this
456 * should not be able to happen.
457 * Number of available devices, which are contained in
458 * device_allocation, is therefore decremented by not doing
459 * an up(device_allocation).
464 *device_data = local_device_data;
469 static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
472 struct dma_slave_config mem2cryp = {
473 .direction = DMA_MEM_TO_DEV,
474 .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
475 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
478 struct dma_slave_config cryp2mem = {
479 .direction = DMA_DEV_TO_MEM,
480 .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
481 .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
485 dma_cap_zero(device_data->dma.mask);
486 dma_cap_set(DMA_SLAVE, device_data->dma.mask);
488 device_data->dma.cfg_mem2cryp = mem_to_engine;
489 device_data->dma.chan_mem2cryp =
490 dma_request_channel(device_data->dma.mask,
492 device_data->dma.cfg_mem2cryp);
494 device_data->dma.cfg_cryp2mem = engine_to_mem;
495 device_data->dma.chan_cryp2mem =
496 dma_request_channel(device_data->dma.mask,
498 device_data->dma.cfg_cryp2mem);
500 dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp);
501 dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem);
503 init_completion(&device_data->dma.cryp_dma_complete);
506 static void cryp_dma_out_callback(void *data)
508 struct cryp_ctx *ctx = (struct cryp_ctx *) data;
509 dev_dbg(ctx->device->dev, "[%s]: ", __func__);
511 complete(&ctx->device->dma.cryp_dma_complete);
514 static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
515 struct scatterlist *sg,
517 enum dma_data_direction direction)
519 struct dma_async_tx_descriptor *desc;
520 struct dma_chan *channel = NULL;
523 dev_dbg(ctx->device->dev, "[%s]: ", __func__);
525 if (unlikely(!IS_ALIGNED((unsigned long)sg, 4))) {
526 dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
527 "aligned! Addr: 0x%08lx", __func__, (unsigned long)sg);
533 channel = ctx->device->dma.chan_mem2cryp;
534 ctx->device->dma.sg_src = sg;
535 ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev,
536 ctx->device->dma.sg_src,
537 ctx->device->dma.nents_src,
540 if (!ctx->device->dma.sg_src_len) {
541 dev_dbg(ctx->device->dev,
542 "[%s]: Could not map the sg list (TO_DEVICE)",
547 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
548 "(TO_DEVICE)", __func__);
550 desc = dmaengine_prep_slave_sg(channel,
551 ctx->device->dma.sg_src,
552 ctx->device->dma.sg_src_len,
553 DMA_MEM_TO_DEV, DMA_CTRL_ACK);
556 case DMA_FROM_DEVICE:
557 channel = ctx->device->dma.chan_cryp2mem;
558 ctx->device->dma.sg_dst = sg;
559 ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev,
560 ctx->device->dma.sg_dst,
561 ctx->device->dma.nents_dst,
564 if (!ctx->device->dma.sg_dst_len) {
565 dev_dbg(ctx->device->dev,
566 "[%s]: Could not map the sg list (FROM_DEVICE)",
571 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
572 "(FROM_DEVICE)", __func__);
574 desc = dmaengine_prep_slave_sg(channel,
575 ctx->device->dma.sg_dst,
576 ctx->device->dma.sg_dst_len,
581 desc->callback = cryp_dma_out_callback;
582 desc->callback_param = ctx;
586 dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction",
591 cookie = dmaengine_submit(desc);
592 if (dma_submit_error(cookie)) {
593 dev_dbg(ctx->device->dev, "[%s]: DMA submission failed\n",
598 dma_async_issue_pending(channel);
603 static void cryp_dma_done(struct cryp_ctx *ctx)
605 struct dma_chan *chan;
607 dev_dbg(ctx->device->dev, "[%s]: ", __func__);
609 chan = ctx->device->dma.chan_mem2cryp;
610 dmaengine_terminate_all(chan);
611 dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src,
612 ctx->device->dma.nents_src, DMA_TO_DEVICE);
614 chan = ctx->device->dma.chan_cryp2mem;
615 dmaengine_terminate_all(chan);
616 dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst,
617 ctx->device->dma.nents_dst, DMA_FROM_DEVICE);
620 static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg,
623 int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
624 dev_dbg(ctx->device->dev, "[%s]: ", __func__);
627 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
635 static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len)
637 int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE);
639 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
647 static void cryp_polling_mode(struct cryp_ctx *ctx,
648 struct cryp_device_data *device_data)
650 int len = ctx->blocksize / BYTES_PER_WORD;
651 int remaining_length = ctx->datalen;
652 u32 *indata = (u32 *)ctx->indata;
653 u32 *outdata = (u32 *)ctx->outdata;
655 while (remaining_length > 0) {
656 writesl(&device_data->base->din, indata, len);
658 remaining_length -= (len * BYTES_PER_WORD);
659 cryp_wait_until_done(device_data);
661 readsl(&device_data->base->dout, outdata, len);
663 cryp_wait_until_done(device_data);
667 static int cryp_disable_power(struct device *dev,
668 struct cryp_device_data *device_data,
669 bool save_device_context)
673 dev_dbg(dev, "[%s]", __func__);
675 spin_lock(&device_data->power_state_spinlock);
676 if (!device_data->power_state)
679 spin_lock(&device_data->ctx_lock);
680 if (save_device_context && device_data->current_ctx) {
681 cryp_save_device_context(device_data,
682 &device_data->current_ctx->dev_ctx,
684 device_data->restore_dev_ctx = true;
686 spin_unlock(&device_data->ctx_lock);
688 clk_disable(device_data->clk);
689 ret = regulator_disable(device_data->pwr_regulator);
691 dev_err(dev, "[%s]: "
692 "regulator_disable() failed!",
695 device_data->power_state = false;
698 spin_unlock(&device_data->power_state_spinlock);
703 static int cryp_enable_power(
705 struct cryp_device_data *device_data,
706 bool restore_device_context)
710 dev_dbg(dev, "[%s]", __func__);
712 spin_lock(&device_data->power_state_spinlock);
713 if (!device_data->power_state) {
714 ret = regulator_enable(device_data->pwr_regulator);
716 dev_err(dev, "[%s]: regulator_enable() failed!",
721 ret = clk_enable(device_data->clk);
723 dev_err(dev, "[%s]: clk_enable() failed!",
725 regulator_disable(device_data->pwr_regulator);
728 device_data->power_state = true;
731 if (device_data->restore_dev_ctx) {
732 spin_lock(&device_data->ctx_lock);
733 if (restore_device_context && device_data->current_ctx) {
734 device_data->restore_dev_ctx = false;
735 cryp_restore_device_context(device_data,
736 &device_data->current_ctx->dev_ctx);
738 spin_unlock(&device_data->ctx_lock);
741 spin_unlock(&device_data->power_state_spinlock);
746 static int hw_crypt_noxts(struct cryp_ctx *ctx,
747 struct cryp_device_data *device_data)
751 const u8 *indata = ctx->indata;
752 u8 *outdata = ctx->outdata;
753 u32 datalen = ctx->datalen;
754 u32 outlen = datalen;
756 pr_debug(DEV_DBG_NAME " [%s]", __func__);
758 ctx->outlen = ctx->datalen;
760 if (unlikely(!IS_ALIGNED((unsigned long)indata, 4))) {
761 pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
762 "0x%08lx", __func__, (unsigned long)indata);
766 ret = cryp_setup_context(ctx, device_data);
771 if (cryp_mode == CRYP_MODE_INTERRUPT) {
772 cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO |
773 CRYP_IRQ_SRC_OUTPUT_FIFO);
776 * ctx->outlen is decremented in the cryp_interrupt_handler
777 * function. We had to add cpu_relax() (barrier) to make sure
778 * that gcc didn't optimze away this variable.
780 while (ctx->outlen > 0)
782 } else if (cryp_mode == CRYP_MODE_POLLING ||
783 cryp_mode == CRYP_MODE_DMA) {
785 * The reason for having DMA in this if case is that if we are
786 * running cryp_mode = 2, then we separate DMA routines for
787 * handling cipher/plaintext > blocksize, except when
788 * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use
789 * the polling mode. Overhead of doing DMA setup eats up the
792 cryp_polling_mode(ctx, device_data);
794 dev_err(ctx->device->dev, "[%s]: Invalid operation mode!",
800 cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
804 ctx->indata = indata;
805 ctx->outdata = outdata;
806 ctx->datalen = datalen;
807 ctx->outlen = outlen;
812 static int get_nents(struct scatterlist *sg, int nbytes)
817 nbytes -= sg->length;
825 static int ablk_dma_crypt(struct skcipher_request *areq)
827 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
828 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
829 struct cryp_device_data *device_data;
831 int bytes_written = 0;
835 pr_debug(DEV_DBG_NAME " [%s]", __func__);
837 ctx->datalen = areq->cryptlen;
838 ctx->outlen = areq->cryptlen;
840 ret = cryp_get_device_data(ctx, &device_data);
844 ret = cryp_setup_context(ctx, device_data);
848 /* We have the device now, so store the nents in the dma struct. */
849 ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen);
850 ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen);
852 /* Enable DMA in- and output. */
853 cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS);
855 bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen);
856 bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written);
858 wait_for_completion(&ctx->device->dma.cryp_dma_complete);
861 cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
865 spin_lock(&device_data->ctx_lock);
866 device_data->current_ctx = NULL;
868 spin_unlock(&device_data->ctx_lock);
871 * The down_interruptible part for this semaphore is called in
872 * cryp_get_device_data.
874 up(&driver_data.device_allocation);
876 if (unlikely(bytes_written != bytes_read))
882 static int ablk_crypt(struct skcipher_request *areq)
884 struct skcipher_walk walk;
885 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
886 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
887 struct cryp_device_data *device_data;
888 unsigned long src_paddr;
889 unsigned long dst_paddr;
893 pr_debug(DEV_DBG_NAME " [%s]", __func__);
895 ret = cryp_get_device_data(ctx, &device_data);
899 ret = skcipher_walk_async(&walk, areq);
902 pr_err(DEV_DBG_NAME "[%s]: skcipher_walk_async() failed!",
907 while ((nbytes = walk.nbytes) > 0) {
909 src_paddr = (page_to_phys(walk.src.phys.page) + walk.src.phys.offset);
910 ctx->indata = phys_to_virt(src_paddr);
912 dst_paddr = (page_to_phys(walk.dst.phys.page) + walk.dst.phys.offset);
913 ctx->outdata = phys_to_virt(dst_paddr);
915 ctx->datalen = nbytes - (nbytes % ctx->blocksize);
917 ret = hw_crypt_noxts(ctx, device_data);
921 nbytes -= ctx->datalen;
922 ret = skcipher_walk_done(&walk, nbytes);
928 /* Release the device */
929 spin_lock(&device_data->ctx_lock);
930 device_data->current_ctx = NULL;
932 spin_unlock(&device_data->ctx_lock);
935 * The down_interruptible part for this semaphore is called in
936 * cryp_get_device_data.
938 up(&driver_data.device_allocation);
943 static int aes_skcipher_setkey(struct crypto_skcipher *cipher,
944 const u8 *key, unsigned int keylen)
946 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
948 pr_debug(DEV_DBG_NAME " [%s]", __func__);
951 case AES_KEYSIZE_128:
952 ctx->config.keysize = CRYP_KEY_SIZE_128;
955 case AES_KEYSIZE_192:
956 ctx->config.keysize = CRYP_KEY_SIZE_192;
959 case AES_KEYSIZE_256:
960 ctx->config.keysize = CRYP_KEY_SIZE_256;
964 pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__);
968 memcpy(ctx->key, key, keylen);
969 ctx->keylen = keylen;
976 static int des_skcipher_setkey(struct crypto_skcipher *cipher,
977 const u8 *key, unsigned int keylen)
979 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
982 pr_debug(DEV_DBG_NAME " [%s]", __func__);
984 err = verify_skcipher_des_key(cipher, key);
988 memcpy(ctx->key, key, keylen);
989 ctx->keylen = keylen;
995 static int des3_skcipher_setkey(struct crypto_skcipher *cipher,
996 const u8 *key, unsigned int keylen)
998 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
1001 pr_debug(DEV_DBG_NAME " [%s]", __func__);
1003 err = verify_skcipher_des3_key(cipher, key);
1007 memcpy(ctx->key, key, keylen);
1008 ctx->keylen = keylen;
1014 static int cryp_blk_encrypt(struct skcipher_request *areq)
1016 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1017 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
1019 pr_debug(DEV_DBG_NAME " [%s]", __func__);
1021 ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT;
1024 * DMA does not work for DES due to a hw bug */
1025 if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1026 return ablk_dma_crypt(areq);
1028 /* For everything except DMA, we run the non DMA version. */
1029 return ablk_crypt(areq);
1032 static int cryp_blk_decrypt(struct skcipher_request *areq)
1034 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1035 struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
1037 pr_debug(DEV_DBG_NAME " [%s]", __func__);
1039 ctx->config.algodir = CRYP_ALGORITHM_DECRYPT;
1041 /* DMA does not work for DES due to a hw bug */
1042 if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1043 return ablk_dma_crypt(areq);
1045 /* For everything except DMA, we run the non DMA version. */
1046 return ablk_crypt(areq);
1049 struct cryp_algo_template {
1050 enum cryp_algo_mode algomode;
1051 struct skcipher_alg skcipher;
1054 static int cryp_init_tfm(struct crypto_skcipher *tfm)
1056 struct cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
1057 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1058 struct cryp_algo_template *cryp_alg = container_of(alg,
1059 struct cryp_algo_template,
1062 ctx->config.algomode = cryp_alg->algomode;
1063 ctx->blocksize = crypto_skcipher_blocksize(tfm);
1068 static struct cryp_algo_template cryp_algs[] = {
1070 .algomode = CRYP_ALGO_AES_ECB,
1072 .base.cra_name = "ecb(aes)",
1073 .base.cra_driver_name = "ecb-aes-ux500",
1074 .base.cra_priority = 300,
1075 .base.cra_flags = CRYPTO_ALG_ASYNC,
1076 .base.cra_blocksize = AES_BLOCK_SIZE,
1077 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1078 .base.cra_alignmask = 3,
1079 .base.cra_module = THIS_MODULE,
1081 .min_keysize = AES_MIN_KEY_SIZE,
1082 .max_keysize = AES_MAX_KEY_SIZE,
1083 .setkey = aes_skcipher_setkey,
1084 .encrypt = cryp_blk_encrypt,
1085 .decrypt = cryp_blk_decrypt,
1086 .init = cryp_init_tfm,
1090 .algomode = CRYP_ALGO_AES_CBC,
1092 .base.cra_name = "cbc(aes)",
1093 .base.cra_driver_name = "cbc-aes-ux500",
1094 .base.cra_priority = 300,
1095 .base.cra_flags = CRYPTO_ALG_ASYNC,
1096 .base.cra_blocksize = AES_BLOCK_SIZE,
1097 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1098 .base.cra_alignmask = 3,
1099 .base.cra_module = THIS_MODULE,
1101 .min_keysize = AES_MIN_KEY_SIZE,
1102 .max_keysize = AES_MAX_KEY_SIZE,
1103 .setkey = aes_skcipher_setkey,
1104 .encrypt = cryp_blk_encrypt,
1105 .decrypt = cryp_blk_decrypt,
1106 .init = cryp_init_tfm,
1107 .ivsize = AES_BLOCK_SIZE,
1111 .algomode = CRYP_ALGO_AES_CTR,
1113 .base.cra_name = "ctr(aes)",
1114 .base.cra_driver_name = "ctr-aes-ux500",
1115 .base.cra_priority = 300,
1116 .base.cra_flags = CRYPTO_ALG_ASYNC,
1117 .base.cra_blocksize = 1,
1118 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1119 .base.cra_alignmask = 3,
1120 .base.cra_module = THIS_MODULE,
1122 .min_keysize = AES_MIN_KEY_SIZE,
1123 .max_keysize = AES_MAX_KEY_SIZE,
1124 .setkey = aes_skcipher_setkey,
1125 .encrypt = cryp_blk_encrypt,
1126 .decrypt = cryp_blk_decrypt,
1127 .init = cryp_init_tfm,
1128 .ivsize = AES_BLOCK_SIZE,
1129 .chunksize = AES_BLOCK_SIZE,
1133 .algomode = CRYP_ALGO_DES_ECB,
1135 .base.cra_name = "ecb(des)",
1136 .base.cra_driver_name = "ecb-des-ux500",
1137 .base.cra_priority = 300,
1138 .base.cra_flags = CRYPTO_ALG_ASYNC,
1139 .base.cra_blocksize = DES_BLOCK_SIZE,
1140 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1141 .base.cra_alignmask = 3,
1142 .base.cra_module = THIS_MODULE,
1144 .min_keysize = DES_KEY_SIZE,
1145 .max_keysize = DES_KEY_SIZE,
1146 .setkey = des_skcipher_setkey,
1147 .encrypt = cryp_blk_encrypt,
1148 .decrypt = cryp_blk_decrypt,
1149 .init = cryp_init_tfm,
1153 .algomode = CRYP_ALGO_TDES_ECB,
1155 .base.cra_name = "ecb(des3_ede)",
1156 .base.cra_driver_name = "ecb-des3_ede-ux500",
1157 .base.cra_priority = 300,
1158 .base.cra_flags = CRYPTO_ALG_ASYNC,
1159 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1160 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1161 .base.cra_alignmask = 3,
1162 .base.cra_module = THIS_MODULE,
1164 .min_keysize = DES3_EDE_KEY_SIZE,
1165 .max_keysize = DES3_EDE_KEY_SIZE,
1166 .setkey = des3_skcipher_setkey,
1167 .encrypt = cryp_blk_encrypt,
1168 .decrypt = cryp_blk_decrypt,
1169 .init = cryp_init_tfm,
1173 .algomode = CRYP_ALGO_DES_CBC,
1175 .base.cra_name = "cbc(des)",
1176 .base.cra_driver_name = "cbc-des-ux500",
1177 .base.cra_priority = 300,
1178 .base.cra_flags = CRYPTO_ALG_ASYNC,
1179 .base.cra_blocksize = DES_BLOCK_SIZE,
1180 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1181 .base.cra_alignmask = 3,
1182 .base.cra_module = THIS_MODULE,
1184 .min_keysize = DES_KEY_SIZE,
1185 .max_keysize = DES_KEY_SIZE,
1186 .setkey = des_skcipher_setkey,
1187 .encrypt = cryp_blk_encrypt,
1188 .decrypt = cryp_blk_decrypt,
1189 .ivsize = DES_BLOCK_SIZE,
1190 .init = cryp_init_tfm,
1194 .algomode = CRYP_ALGO_TDES_CBC,
1196 .base.cra_name = "cbc(des3_ede)",
1197 .base.cra_driver_name = "cbc-des3_ede-ux500",
1198 .base.cra_priority = 300,
1199 .base.cra_flags = CRYPTO_ALG_ASYNC,
1200 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1201 .base.cra_ctxsize = sizeof(struct cryp_ctx),
1202 .base.cra_alignmask = 3,
1203 .base.cra_module = THIS_MODULE,
1205 .min_keysize = DES3_EDE_KEY_SIZE,
1206 .max_keysize = DES3_EDE_KEY_SIZE,
1207 .setkey = des3_skcipher_setkey,
1208 .encrypt = cryp_blk_encrypt,
1209 .decrypt = cryp_blk_decrypt,
1210 .ivsize = DES3_EDE_BLOCK_SIZE,
1211 .init = cryp_init_tfm,
1217 * cryp_algs_register_all -
1219 static int cryp_algs_register_all(void)
1225 pr_debug("[%s]", __func__);
1227 for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) {
1228 ret = crypto_register_skcipher(&cryp_algs[i].skcipher);
1231 pr_err("[%s] alg registration failed",
1232 cryp_algs[i].skcipher.base.cra_driver_name);
1238 for (i = 0; i < count; i++)
1239 crypto_unregister_skcipher(&cryp_algs[i].skcipher);
1244 * cryp_algs_unregister_all -
1246 static void cryp_algs_unregister_all(void)
1250 pr_debug(DEV_DBG_NAME " [%s]", __func__);
1252 for (i = 0; i < ARRAY_SIZE(cryp_algs); i++)
1253 crypto_unregister_skcipher(&cryp_algs[i].skcipher);
1256 static int ux500_cryp_probe(struct platform_device *pdev)
1259 struct resource *res;
1260 struct resource *res_irq;
1261 struct cryp_device_data *device_data;
1262 struct cryp_protection_config prot = {
1263 .privilege_access = CRYP_STATE_ENABLE
1265 struct device *dev = &pdev->dev;
1267 dev_dbg(dev, "[%s]", __func__);
1268 device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC);
1274 device_data->dev = dev;
1275 device_data->current_ctx = NULL;
1277 /* Grab the DMA configuration from platform data. */
1278 mem_to_engine = &((struct cryp_platform_data *)
1279 dev->platform_data)->mem_to_engine;
1280 engine_to_mem = &((struct cryp_platform_data *)
1281 dev->platform_data)->engine_to_mem;
1283 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1285 dev_err(dev, "[%s]: platform_get_resource() failed",
1291 device_data->phybase = res->start;
1292 device_data->base = devm_ioremap_resource(dev, res);
1293 if (IS_ERR(device_data->base)) {
1294 ret = PTR_ERR(device_data->base);
1298 spin_lock_init(&device_data->ctx_lock);
1299 spin_lock_init(&device_data->power_state_spinlock);
1301 /* Enable power for CRYP hardware block */
1302 device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape");
1303 if (IS_ERR(device_data->pwr_regulator)) {
1304 dev_err(dev, "[%s]: could not get cryp regulator", __func__);
1305 ret = PTR_ERR(device_data->pwr_regulator);
1306 device_data->pwr_regulator = NULL;
1310 /* Enable the clk for CRYP hardware block */
1311 device_data->clk = devm_clk_get(&pdev->dev, NULL);
1312 if (IS_ERR(device_data->clk)) {
1313 dev_err(dev, "[%s]: clk_get() failed!", __func__);
1314 ret = PTR_ERR(device_data->clk);
1318 ret = clk_prepare(device_data->clk);
1320 dev_err(dev, "[%s]: clk_prepare() failed!", __func__);
1324 /* Enable device power (and clock) */
1325 ret = cryp_enable_power(device_data->dev, device_data, false);
1327 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1328 goto out_clk_unprepare;
1331 if (cryp_check(device_data)) {
1332 dev_err(dev, "[%s]: cryp_check() failed!", __func__);
1337 if (cryp_configure_protection(device_data, &prot)) {
1338 dev_err(dev, "[%s]: cryp_configure_protection() failed!",
1344 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1346 dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
1352 ret = devm_request_irq(&pdev->dev, res_irq->start,
1353 cryp_interrupt_handler, 0, "cryp1", device_data);
1355 dev_err(dev, "[%s]: Unable to request IRQ", __func__);
1359 if (cryp_mode == CRYP_MODE_DMA)
1360 cryp_dma_setup_channel(device_data, dev);
1362 platform_set_drvdata(pdev, device_data);
1364 /* Put the new device into the device list... */
1365 klist_add_tail(&device_data->list_node, &driver_data.device_list);
1367 /* ... and signal that a new device is available. */
1368 up(&driver_data.device_allocation);
1370 atomic_set(&session_id, 1);
1372 ret = cryp_algs_register_all();
1374 dev_err(dev, "[%s]: cryp_algs_register_all() failed!",
1379 dev_info(dev, "successfully registered\n");
1384 cryp_disable_power(device_data->dev, device_data, false);
1387 clk_unprepare(device_data->clk);
1390 regulator_put(device_data->pwr_regulator);
1396 static int ux500_cryp_remove(struct platform_device *pdev)
1398 struct cryp_device_data *device_data;
1400 dev_dbg(&pdev->dev, "[%s]", __func__);
1401 device_data = platform_get_drvdata(pdev);
1403 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1408 /* Try to decrease the number of available devices. */
1409 if (down_trylock(&driver_data.device_allocation))
1412 /* Check that the device is free */
1413 spin_lock(&device_data->ctx_lock);
1414 /* current_ctx allocates a device, NULL = unallocated */
1415 if (device_data->current_ctx) {
1416 /* The device is busy */
1417 spin_unlock(&device_data->ctx_lock);
1418 /* Return the device to the pool. */
1419 up(&driver_data.device_allocation);
1423 spin_unlock(&device_data->ctx_lock);
1425 /* Remove the device from the list */
1426 if (klist_node_attached(&device_data->list_node))
1427 klist_remove(&device_data->list_node);
1429 /* If this was the last device, remove the services */
1430 if (list_empty(&driver_data.device_list.k_list))
1431 cryp_algs_unregister_all();
1433 if (cryp_disable_power(&pdev->dev, device_data, false))
1434 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1437 clk_unprepare(device_data->clk);
1438 regulator_put(device_data->pwr_regulator);
1443 static void ux500_cryp_shutdown(struct platform_device *pdev)
1445 struct cryp_device_data *device_data;
1447 dev_dbg(&pdev->dev, "[%s]", __func__);
1449 device_data = platform_get_drvdata(pdev);
1451 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1456 /* Check that the device is free */
1457 spin_lock(&device_data->ctx_lock);
1458 /* current_ctx allocates a device, NULL = unallocated */
1459 if (!device_data->current_ctx) {
1460 if (down_trylock(&driver_data.device_allocation))
1461 dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
1462 "Shutting down anyway...", __func__);
1464 * (Allocate the device)
1465 * Need to set this to non-null (dummy) value,
1466 * to avoid usage if context switching.
1468 device_data->current_ctx++;
1470 spin_unlock(&device_data->ctx_lock);
1472 /* Remove the device from the list */
1473 if (klist_node_attached(&device_data->list_node))
1474 klist_remove(&device_data->list_node);
1476 /* If this was the last device, remove the services */
1477 if (list_empty(&driver_data.device_list.k_list))
1478 cryp_algs_unregister_all();
1480 if (cryp_disable_power(&pdev->dev, device_data, false))
1481 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1486 #ifdef CONFIG_PM_SLEEP
1487 static int ux500_cryp_suspend(struct device *dev)
1490 struct platform_device *pdev = to_platform_device(dev);
1491 struct cryp_device_data *device_data;
1492 struct resource *res_irq;
1493 struct cryp_ctx *temp_ctx = NULL;
1495 dev_dbg(dev, "[%s]", __func__);
1498 device_data = platform_get_drvdata(pdev);
1500 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1504 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1506 dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
1508 disable_irq(res_irq->start);
1510 spin_lock(&device_data->ctx_lock);
1511 if (!device_data->current_ctx)
1512 device_data->current_ctx++;
1513 spin_unlock(&device_data->ctx_lock);
1515 if (device_data->current_ctx == ++temp_ctx) {
1516 if (down_interruptible(&driver_data.device_allocation))
1517 dev_dbg(dev, "[%s]: down_interruptible() failed",
1519 ret = cryp_disable_power(dev, device_data, false);
1522 ret = cryp_disable_power(dev, device_data, true);
1525 dev_err(dev, "[%s]: cryp_disable_power()", __func__);
1530 static int ux500_cryp_resume(struct device *dev)
1533 struct platform_device *pdev = to_platform_device(dev);
1534 struct cryp_device_data *device_data;
1535 struct resource *res_irq;
1536 struct cryp_ctx *temp_ctx = NULL;
1538 dev_dbg(dev, "[%s]", __func__);
1540 device_data = platform_get_drvdata(pdev);
1542 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1546 spin_lock(&device_data->ctx_lock);
1547 if (device_data->current_ctx == ++temp_ctx)
1548 device_data->current_ctx = NULL;
1549 spin_unlock(&device_data->ctx_lock);
1552 if (!device_data->current_ctx)
1553 up(&driver_data.device_allocation);
1555 ret = cryp_enable_power(dev, device_data, true);
1558 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1560 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1562 enable_irq(res_irq->start);
1569 static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
1571 static const struct of_device_id ux500_cryp_match[] = {
1572 { .compatible = "stericsson,ux500-cryp" },
1575 MODULE_DEVICE_TABLE(of, ux500_cryp_match);
1577 static struct platform_driver cryp_driver = {
1578 .probe = ux500_cryp_probe,
1579 .remove = ux500_cryp_remove,
1580 .shutdown = ux500_cryp_shutdown,
1583 .of_match_table = ux500_cryp_match,
1584 .pm = &ux500_cryp_pm,
1588 static int __init ux500_cryp_mod_init(void)
1590 pr_debug("[%s] is called!", __func__);
1591 klist_init(&driver_data.device_list, NULL, NULL);
1592 /* Initialize the semaphore to 0 devices (locked state) */
1593 sema_init(&driver_data.device_allocation, 0);
1594 return platform_driver_register(&cryp_driver);
1597 static void __exit ux500_cryp_mod_fini(void)
1599 pr_debug("[%s] is called!", __func__);
1600 platform_driver_unregister(&cryp_driver);
1603 module_init(ux500_cryp_mod_init);
1604 module_exit(ux500_cryp_mod_fini);
1606 module_param(cryp_mode, int, 0);
1608 MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine.");
1609 MODULE_ALIAS_CRYPTO("aes-all");
1610 MODULE_ALIAS_CRYPTO("des-all");
1612 MODULE_LICENSE("GPL");