drivers/crypto/ux500/cryp/cryp_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * Copyright (C) ST-Ericsson SA 2010
   4  * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson.
   5  * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson.
   6  * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson.
   7  * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson.
   8  * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson.
   9  * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson.
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/completion.h>
  14 #include <linux/device.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/err.h>
  18 #include <linux/errno.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/io.h>
  21 #include <linux/irqreturn.h>
  22 #include <linux/kernel.h>
  23 #include <linux/klist.h>
  24 #include <linux/module.h>
  25 #include <linux/mod_devicetable.h>
  26 #include <linux/platform_device.h>
  27 #include <linux/regulator/consumer.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/platform_data/dma-ste-dma40.h>
  30
  31 #include <crypto/aes.h>
  32 #include <crypto/ctr.h>
  33 #include <crypto/internal/des.h>
  34 #include <crypto/internal/skcipher.h>
  35 #include <crypto/scatterwalk.h>
  36
  37 #include <linux/platform_data/crypto-ux500.h>
  38
  39 #include "cryp_p.h"
  40 #include "cryp.h"
  41
  42 #define CRYP_MAX_KEY_SIZE       32
  43 #define BYTES_PER_WORD          4
  44
  45 static int cryp_mode;
  46 static atomic_t session_id;
  47
  48 static struct stedma40_chan_cfg *mem_to_engine;
  49 static struct stedma40_chan_cfg *engine_to_mem;
  50
  51 /**
  52  * struct cryp_driver_data - data specific to the driver.
  53  *
  54  * @device_list: A list of registered devices to choose from.
  55  * @device_allocation: A semaphore initialized with number of devices.
  56  */
  57 struct cryp_driver_data {
  58         struct klist device_list;
  59         struct semaphore device_allocation;
  60 };
  61
  62 /**
  63  * struct cryp_ctx - Crypto context
  64  * @config: Crypto mode.
  65  * @key[CRYP_MAX_KEY_SIZE]: Key.
  66  * @keylen: Length of key.
  67  * @iv: Pointer to initialization vector.
  68  * @indata: Pointer to indata.
  69  * @outdata: Pointer to outdata.
  70  * @datalen: Length of indata.
  71  * @outlen: Length of outdata.
  72  * @blocksize: Size of blocks.
  73  * @updated: Updated flag.
  74  * @dev_ctx: Device dependent context.
  75  * @device: Pointer to the device.
  76  */
  77 struct cryp_ctx {
  78         struct cryp_config config;
  79         u8 key[CRYP_MAX_KEY_SIZE];
  80         u32 keylen;
  81         u8 *iv;
  82         const u8 *indata;
  83         u8 *outdata;
  84         u32 datalen;
  85         u32 outlen;
  86         u32 blocksize;
  87         u8 updated;
  88         struct cryp_device_context dev_ctx;
  89         struct cryp_device_data *device;
  90         u32 session_id;
  91 };
  92
  93 static struct cryp_driver_data driver_data;
  94
  95 /**
  96  * swap_bits_in_byte - mirror the bits in a byte
  97  * @b: the byte to be mirrored
  98  *
  99  * The bits are swapped the following way:
 100  *  Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and
 101  *  nibble 2 (n2) bits 4-7.
 102  *
 103  *  Nibble 1 (n1):
 104  *  (The "old" (moved) bit is replaced with a zero)
 105  *  1. Move bit 6 and 7, 4 positions to the left.
 106  *  2. Move bit 3 and 5, 2 positions to the left.
 107  *  3. Move bit 1-4, 1 position to the left.
 108  *
 109  *  Nibble 2 (n2):
 110  *  1. Move bit 0 and 1, 4 positions to the right.
 111  *  2. Move bit 2 and 4, 2 positions to the right.
 112  *  3. Move bit 3-6, 1 position to the right.
 113  *
 114  *  Combine the two nibbles to a complete and swapped byte.
 115  */
 116
 117 static inline u8 swap_bits_in_byte(u8 b)
 118 {
 119 #define R_SHIFT_4_MASK  0xc0 /* Bits 6 and 7, right shift 4 */
 120 #define R_SHIFT_2_MASK  0x28 /* (After right shift 4) Bits 3 and 5,
 121                                   right shift 2 */
 122 #define R_SHIFT_1_MASK  0x1e /* (After right shift 2) Bits 1-4,
 123                                   right shift 1 */
 124 #define L_SHIFT_4_MASK  0x03 /* Bits 0 and 1, left shift 4 */
 125 #define L_SHIFT_2_MASK  0x14 /* (After left shift 4) Bits 2 and 4,
 126                                   left shift 2 */
 127 #define L_SHIFT_1_MASK  0x78 /* (After left shift 1) Bits 3-6,
 128                                   left shift 1 */
 129
 130         u8 n1;
 131         u8 n2;
 132
 133         /* Swap most significant nibble */
 134         /* Right shift 4, bits 6 and 7 */
 135         n1 = ((b  & R_SHIFT_4_MASK) >> 4) | (b  & ~(R_SHIFT_4_MASK >> 4));
 136         /* Right shift 2, bits 3 and 5 */
 137         n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));
 138         /* Right shift 1, bits 1-4 */
 139         n1 = (n1  & R_SHIFT_1_MASK) >> 1;
 140
 141         /* Swap least significant nibble */
 142         /* Left shift 4, bits 0 and 1 */
 143         n2 = ((b  & L_SHIFT_4_MASK) << 4) | (b  & ~(L_SHIFT_4_MASK << 4));
 144         /* Left shift 2, bits 2 and 4 */
 145         n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));
 146         /* Left shift 1, bits 3-6 */
 147         n2 = (n2  & L_SHIFT_1_MASK) << 1;
 148
 149         return n1 | n2;
 150 }
 151
 152 static inline void swap_words_in_key_and_bits_in_byte(const u8 *in,
 153                                                       u8 *out, u32 len)
 154 {
 155         unsigned int i = 0;
 156         int j;
 157         int index = 0;
 158
 159         j = len - BYTES_PER_WORD;
 160         while (j >= 0) {
 161                 for (i = 0; i < BYTES_PER_WORD; i++) {
 162                         index = len - j - BYTES_PER_WORD + i;
 163                         out[j + i] =
 164                                 swap_bits_in_byte(in[index]);
 165                 }
 166                 j -= BYTES_PER_WORD;
 167         }
 168 }
 169
 170 static void add_session_id(struct cryp_ctx *ctx)
 171 {
 172         /*
 173          * We never want 0 to be a valid value, since this is the default value
 174          * for the software context.
 175          */
 176         if (unlikely(atomic_inc_and_test(&session_id)))
 177                 atomic_inc(&session_id);
 178
 179         ctx->session_id = atomic_read(&session_id);
 180 }
 181
 182 static irqreturn_t cryp_interrupt_handler(int irq, void *param)
 183 {
 184         struct cryp_ctx *ctx;
 185         int count;
 186         struct cryp_device_data *device_data;
 187
 188         if (param == NULL) {
 189                 BUG_ON(!param);
 190                 return IRQ_HANDLED;
 191         }
 192
 193         /* The device is coming from the one found in hw_crypt_noxts. */
 194         device_data = (struct cryp_device_data *)param;
 195
 196         ctx = device_data->current_ctx;
 197
 198         if (ctx == NULL) {
 199                 BUG_ON(!ctx);
 200                 return IRQ_HANDLED;
 201         }
 202
 203         dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen,
 204                 cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ?
 205                 "out" : "in");
 206
 207         if (cryp_pending_irq_src(device_data,
 208                                  CRYP_IRQ_SRC_OUTPUT_FIFO)) {
 209                 if (ctx->outlen / ctx->blocksize > 0) {
 210                         count = ctx->blocksize / 4;
 211
 212                         readsl(&device_data->base->dout, ctx->outdata, count);
 213                         ctx->outdata += count;
 214                         ctx->outlen -= count;
 215
 216                         if (ctx->outlen == 0) {
 217                                 cryp_disable_irq_src(device_data,
 218                                                      CRYP_IRQ_SRC_OUTPUT_FIFO);
 219                         }
 220                 }
 221         } else if (cryp_pending_irq_src(device_data,
 222                                         CRYP_IRQ_SRC_INPUT_FIFO)) {
 223                 if (ctx->datalen / ctx->blocksize > 0) {
 224                         count = ctx->blocksize / 4;
 225
 226                         writesl(&device_data->base->din, ctx->indata, count);
 227
 228                         ctx->indata += count;
 229                         ctx->datalen -= count;
 230
 231                         if (ctx->datalen == 0)
 232                                 cryp_disable_irq_src(device_data,
 233                                                    CRYP_IRQ_SRC_INPUT_FIFO);
 234
 235                         if (ctx->config.algomode == CRYP_ALGO_AES_XTS) {
 236                                 CRYP_PUT_BITS(&device_data->base->cr,
 237                                               CRYP_START_ENABLE,
 238                                               CRYP_CR_START_POS,
 239                                               CRYP_CR_START_MASK);
 240
 241                                 cryp_wait_until_done(device_data);
 242                         }
 243                 }
 244         }
 245
 246         return IRQ_HANDLED;
 247 }
 248
 249 static int mode_is_aes(enum cryp_algo_mode mode)
 250 {
 251         return  CRYP_ALGO_AES_ECB == mode ||
 252                 CRYP_ALGO_AES_CBC == mode ||
 253                 CRYP_ALGO_AES_CTR == mode ||
 254                 CRYP_ALGO_AES_XTS == mode;
 255 }
 256
 257 static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right,
 258                   enum cryp_init_vector_index index)
 259 {
 260         struct cryp_init_vector_value vector_value;
 261
 262         dev_dbg(device_data->dev, "[%s]", __func__);
 263
 264         vector_value.init_value_left = left;
 265         vector_value.init_value_right = right;
 266
 267         return cryp_configure_init_vector(device_data,
 268                                           index,
 269                                           vector_value);
 270 }
 271
 272 static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx)
 273 {
 274         int i;
 275         int status = 0;
 276         int num_of_regs = ctx->blocksize / 8;
 277         __be32 *civ = (__be32 *)ctx->iv;
 278         u32 iv[AES_BLOCK_SIZE / 4];
 279
 280         dev_dbg(device_data->dev, "[%s]", __func__);
 281
 282         /*
 283          * Since we loop on num_of_regs we need to have a check in case
 284          * someone provides an incorrect blocksize which would force calling
 285          * cfg_iv with i greater than 2 which is an error.
 286          */
 287         if (num_of_regs > 2) {
 288                 dev_err(device_data->dev, "[%s] Incorrect blocksize %d",
 289                         __func__, ctx->blocksize);
 290                 return -EINVAL;
 291         }
 292
 293         for (i = 0; i < ctx->blocksize / 4; i++)
 294                 iv[i] = be32_to_cpup(civ + i);
 295
 296         for (i = 0; i < num_of_regs; i++) {
 297                 status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
 298                                 (enum cryp_init_vector_index) i);
 299                 if (status != 0)
 300                         return status;
 301         }
 302         return status;
 303 }
 304
 305 static int set_key(struct cryp_device_data *device_data,
 306                    u32 left_key,
 307                    u32 right_key,
 308                    enum cryp_key_reg_index index)
 309 {
 310         struct cryp_key_value key_value;
 311         int cryp_error;
 312
 313         dev_dbg(device_data->dev, "[%s]", __func__);
 314
 315         key_value.key_value_left = left_key;
 316         key_value.key_value_right = right_key;
 317
 318         cryp_error = cryp_configure_key_values(device_data,
 319                                                index,
 320                                                key_value);
 321         if (cryp_error != 0)
 322                 dev_err(device_data->dev, "[%s]: "
 323                         "cryp_configure_key_values() failed!", __func__);
 324
 325         return cryp_error;
 326 }
 327
 328 static int cfg_keys(struct cryp_ctx *ctx)
 329 {
 330         int i;
 331         int num_of_regs = ctx->keylen / 8;
 332         u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
 333         __be32 *ckey = (__be32 *)ctx->key;
 334         int cryp_error = 0;
 335
 336         dev_dbg(ctx->device->dev, "[%s]", __func__);
 337
 338         if (mode_is_aes(ctx->config.algomode)) {
 339                 swap_words_in_key_and_bits_in_byte((u8 *)ckey,
 340                                                    (u8 *)swapped_key,
 341                                                    ctx->keylen);
 342         } else {
 343                 for (i = 0; i < ctx->keylen / 4; i++)
 344                         swapped_key[i] = be32_to_cpup(ckey + i);
 345         }
 346
 347         for (i = 0; i < num_of_regs; i++) {
 348                 cryp_error = set_key(ctx->device,
 349                                      swapped_key[i * 2],
 350                                      swapped_key[i * 2 + 1],
 351                                      (enum cryp_key_reg_index) i);
 352
 353                 if (cryp_error != 0) {
 354                         dev_err(ctx->device->dev, "[%s]: set_key() failed!",
 355                                         __func__);
 356                         return cryp_error;
 357                 }
 358         }
 359         return cryp_error;
 360 }
 361
 362 static int cryp_setup_context(struct cryp_ctx *ctx,
 363                               struct cryp_device_data *device_data)
 364 {
 365         u32 control_register = CRYP_CR_DEFAULT;
 366
 367         switch (cryp_mode) {
 368         case CRYP_MODE_INTERRUPT:
 369                 writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc);
 370                 break;
 371
 372         case CRYP_MODE_DMA:
 373                 writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr);
 374                 break;
 375
 376         default:
 377                 break;
 378         }
 379
 380         if (ctx->updated == 0) {
 381                 cryp_flush_inoutfifo(device_data);
 382                 if (cfg_keys(ctx) != 0) {
 383                         dev_err(ctx->device->dev, "[%s]: cfg_keys failed!",
 384                                 __func__);
 385                         return -EINVAL;
 386                 }
 387
 388                 if (ctx->iv &&
 389                     CRYP_ALGO_AES_ECB != ctx->config.algomode &&
 390                     CRYP_ALGO_DES_ECB != ctx->config.algomode &&
 391                     CRYP_ALGO_TDES_ECB != ctx->config.algomode) {
 392                         if (cfg_ivs(device_data, ctx) != 0)
 393                                 return -EPERM;
 394                 }
 395
 396                 cryp_set_configuration(device_data, &ctx->config,
 397                                        &control_register);
 398                 add_session_id(ctx);
 399         } else if (ctx->updated == 1 &&
 400                    ctx->session_id != atomic_read(&session_id)) {
 401                 cryp_flush_inoutfifo(device_data);
 402                 cryp_restore_device_context(device_data, &ctx->dev_ctx);
 403
 404                 add_session_id(ctx);
 405                 control_register = ctx->dev_ctx.cr;
 406         } else
 407                 control_register = ctx->dev_ctx.cr;
 408
 409         writel(control_register |
 410                (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS),
 411                &device_data->base->cr);
 412
 413         return 0;
 414 }
 415
 416 static int cryp_get_device_data(struct cryp_ctx *ctx,
 417                                 struct cryp_device_data **device_data)
 418 {
 419         int ret;
 420         struct klist_iter device_iterator;
 421         struct klist_node *device_node;
 422         struct cryp_device_data *local_device_data = NULL;
 423         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 424
 425         /* Wait until a device is available */
 426         ret = down_interruptible(&driver_data.device_allocation);
 427         if (ret)
 428                 return ret;  /* Interrupted */
 429
 430         /* Select a device */
 431         klist_iter_init(&driver_data.device_list, &device_iterator);
 432
 433         device_node = klist_next(&device_iterator);
 434         while (device_node) {
 435                 local_device_data = container_of(device_node,
 436                                            struct cryp_device_data, list_node);
 437                 spin_lock(&local_device_data->ctx_lock);
 438                 /* current_ctx allocates a device, NULL = unallocated */
 439                 if (local_device_data->current_ctx) {
 440                         device_node = klist_next(&device_iterator);
 441                 } else {
 442                         local_device_data->current_ctx = ctx;
 443                         ctx->device = local_device_data;
 444                         spin_unlock(&local_device_data->ctx_lock);
 445                         break;
 446                 }
 447                 spin_unlock(&local_device_data->ctx_lock);
 448         }
 449         klist_iter_exit(&device_iterator);
 450
 451         if (!device_node) {
 452                 /**
 453                  * No free device found.
 454                  * Since we allocated a device with down_interruptible, this
 455                  * should not be able to happen.
 456                  * Number of available devices, which are contained in
 457                  * device_allocation, is therefore decremented by not doing
 458                  * an up(device_allocation).
 459                  */
 460                 return -EBUSY;
 461         }
 462
 463         *device_data = local_device_data;
 464
 465         return 0;
 466 }
 467
 468 static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
 469                                    struct device *dev)
 470 {
 471         struct dma_slave_config mem2cryp = {
 472                 .direction = DMA_MEM_TO_DEV,
 473                 .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
 474                 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
 475                 .dst_maxburst = 4,
 476         };
 477         struct dma_slave_config cryp2mem = {
 478                 .direction = DMA_DEV_TO_MEM,
 479                 .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
 480                 .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
 481                 .src_maxburst = 4,
 482         };
 483
 484         dma_cap_zero(device_data->dma.mask);
 485         dma_cap_set(DMA_SLAVE, device_data->dma.mask);
 486
 487         device_data->dma.cfg_mem2cryp = mem_to_engine;
 488         device_data->dma.chan_mem2cryp =
 489                 dma_request_channel(device_data->dma.mask,
 490                                     stedma40_filter,
 491                                     device_data->dma.cfg_mem2cryp);
 492
 493         device_data->dma.cfg_cryp2mem = engine_to_mem;
 494         device_data->dma.chan_cryp2mem =
 495                 dma_request_channel(device_data->dma.mask,
 496                                     stedma40_filter,
 497                                     device_data->dma.cfg_cryp2mem);
 498
 499         dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp);
 500         dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem);
 501
 502         init_completion(&device_data->dma.cryp_dma_complete);
 503 }
 504
 505 static void cryp_dma_out_callback(void *data)
 506 {
 507         struct cryp_ctx *ctx = (struct cryp_ctx *) data;
 508         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 509
 510         complete(&ctx->device->dma.cryp_dma_complete);
 511 }
 512
 513 static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
 514                                  struct scatterlist *sg,
 515                                  int len,
 516                                  enum dma_data_direction direction)
 517 {
 518         struct dma_async_tx_descriptor *desc;
 519         struct dma_chan *channel = NULL;
 520         dma_cookie_t cookie;
 521
 522         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 523
 524         if (unlikely(!IS_ALIGNED((unsigned long)sg, 4))) {
 525                 dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
 526                         "aligned! Addr: 0x%08lx", __func__, (unsigned long)sg);
 527                 return -EFAULT;
 528         }
 529
 530         switch (direction) {
 531         case DMA_TO_DEVICE:
 532                 channel = ctx->device->dma.chan_mem2cryp;
 533                 ctx->device->dma.sg_src = sg;
 534                 ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev,
 535                                                  ctx->device->dma.sg_src,
 536                                                  ctx->device->dma.nents_src,
 537                                                  direction);
 538
 539                 if (!ctx->device->dma.sg_src_len) {
 540                         dev_dbg(ctx->device->dev,
 541                                 "[%s]: Could not map the sg list (TO_DEVICE)",
 542                                 __func__);
 543                         return -EFAULT;
 544                 }
 545
 546                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 547                         "(TO_DEVICE)", __func__);
 548
 549                 desc = dmaengine_prep_slave_sg(channel,
 550                                 ctx->device->dma.sg_src,
 551                                 ctx->device->dma.sg_src_len,
 552                                 DMA_MEM_TO_DEV, DMA_CTRL_ACK);
 553                 break;
 554
 555         case DMA_FROM_DEVICE:
 556                 channel = ctx->device->dma.chan_cryp2mem;
 557                 ctx->device->dma.sg_dst = sg;
 558                 ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev,
 559                                                  ctx->device->dma.sg_dst,
 560                                                  ctx->device->dma.nents_dst,
 561                                                  direction);
 562
 563                 if (!ctx->device->dma.sg_dst_len) {
 564                         dev_dbg(ctx->device->dev,
 565                                 "[%s]: Could not map the sg list (FROM_DEVICE)",
 566                                 __func__);
 567                         return -EFAULT;
 568                 }
 569
 570                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 571                         "(FROM_DEVICE)", __func__);
 572
 573                 desc = dmaengine_prep_slave_sg(channel,
 574                                 ctx->device->dma.sg_dst,
 575                                 ctx->device->dma.sg_dst_len,
 576                                 DMA_DEV_TO_MEM,
 577                                 DMA_CTRL_ACK |
 578                                 DMA_PREP_INTERRUPT);
 579
 580                 desc->callback = cryp_dma_out_callback;
 581                 desc->callback_param = ctx;
 582                 break;
 583
 584         default:
 585                 dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction",
 586                         __func__);
 587                 return -EFAULT;
 588         }
 589
 590         cookie = dmaengine_submit(desc);
 591         if (dma_submit_error(cookie)) {
 592                 dev_dbg(ctx->device->dev, "[%s]: DMA submission failed\n",
 593                         __func__);
 594                 return cookie;
 595         }
 596
 597         dma_async_issue_pending(channel);
 598
 599         return 0;
 600 }
 601
 602 static void cryp_dma_done(struct cryp_ctx *ctx)
 603 {
 604         struct dma_chan *chan;
 605
 606         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 607
 608         chan = ctx->device->dma.chan_mem2cryp;
 609         dmaengine_terminate_all(chan);
 610         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src,
 611                      ctx->device->dma.sg_src_len, DMA_TO_DEVICE);
 612
 613         chan = ctx->device->dma.chan_cryp2mem;
 614         dmaengine_terminate_all(chan);
 615         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst,
 616                      ctx->device->dma.sg_dst_len, DMA_FROM_DEVICE);
 617 }
 618
 619 static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg,
 620                           int len)
 621 {
 622         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
 623         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 624
 625         if (error) {
 626                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 627                         "failed", __func__);
 628                 return error;
 629         }
 630
 631         return len;
 632 }
 633
 634 static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len)
 635 {
 636         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE);
 637         if (error) {
 638                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 639                         "failed", __func__);
 640                 return error;
 641         }
 642
 643         return len;
 644 }
 645
 646 static void cryp_polling_mode(struct cryp_ctx *ctx,
 647                               struct cryp_device_data *device_data)
 648 {
 649         int len = ctx->blocksize / BYTES_PER_WORD;
 650         int remaining_length = ctx->datalen;
 651         u32 *indata = (u32 *)ctx->indata;
 652         u32 *outdata = (u32 *)ctx->outdata;
 653
 654         while (remaining_length > 0) {
 655                 writesl(&device_data->base->din, indata, len);
 656                 indata += len;
 657                 remaining_length -= (len * BYTES_PER_WORD);
 658                 cryp_wait_until_done(device_data);
 659
 660                 readsl(&device_data->base->dout, outdata, len);
 661                 outdata += len;
 662                 cryp_wait_until_done(device_data);
 663         }
 664 }
 665
 666 static int cryp_disable_power(struct device *dev,
 667                               struct cryp_device_data *device_data,
 668                               bool save_device_context)
 669 {
 670         int ret = 0;
 671
 672         dev_dbg(dev, "[%s]", __func__);
 673
 674         spin_lock(&device_data->power_state_spinlock);
 675         if (!device_data->power_state)
 676                 goto out;
 677
 678         spin_lock(&device_data->ctx_lock);
 679         if (save_device_context && device_data->current_ctx) {
 680                 cryp_save_device_context(device_data,
 681                                 &device_data->current_ctx->dev_ctx,
 682                                 cryp_mode);
 683                 device_data->restore_dev_ctx = true;
 684         }
 685         spin_unlock(&device_data->ctx_lock);
 686
 687         clk_disable(device_data->clk);
 688         ret = regulator_disable(device_data->pwr_regulator);
 689         if (ret)
 690                 dev_err(dev, "[%s]: "
 691                                 "regulator_disable() failed!",
 692                                 __func__);
 693
 694         device_data->power_state = false;
 695
 696 out:
 697         spin_unlock(&device_data->power_state_spinlock);
 698
 699         return ret;
 700 }
 701
 702 static int cryp_enable_power(
 703                 struct device *dev,
 704                 struct cryp_device_data *device_data,
 705                 bool restore_device_context)
 706 {
 707         int ret = 0;
 708
 709         dev_dbg(dev, "[%s]", __func__);
 710
 711         spin_lock(&device_data->power_state_spinlock);
 712         if (!device_data->power_state) {
 713                 ret = regulator_enable(device_data->pwr_regulator);
 714                 if (ret) {
 715                         dev_err(dev, "[%s]: regulator_enable() failed!",
 716                                         __func__);
 717                         goto out;
 718                 }
 719
 720                 ret = clk_enable(device_data->clk);
 721                 if (ret) {
 722                         dev_err(dev, "[%s]: clk_enable() failed!",
 723                                         __func__);
 724                         regulator_disable(device_data->pwr_regulator);
 725                         goto out;
 726                 }
 727                 device_data->power_state = true;
 728         }
 729
 730         if (device_data->restore_dev_ctx) {
 731                 spin_lock(&device_data->ctx_lock);
 732                 if (restore_device_context && device_data->current_ctx) {
 733                         device_data->restore_dev_ctx = false;
 734                         cryp_restore_device_context(device_data,
 735                                         &device_data->current_ctx->dev_ctx);
 736                 }
 737                 spin_unlock(&device_data->ctx_lock);
 738         }
 739 out:
 740         spin_unlock(&device_data->power_state_spinlock);
 741
 742         return ret;
 743 }
 744
 745 static int hw_crypt_noxts(struct cryp_ctx *ctx,
 746                           struct cryp_device_data *device_data)
 747 {
 748         int ret = 0;
 749
 750         const u8 *indata = ctx->indata;
 751         u8 *outdata = ctx->outdata;
 752         u32 datalen = ctx->datalen;
 753         u32 outlen = datalen;
 754
 755         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 756
 757         ctx->outlen = ctx->datalen;
 758
 759         if (unlikely(!IS_ALIGNED((unsigned long)indata, 4))) {
 760                 pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
 761                          "0x%08lx", __func__, (unsigned long)indata);
 762                 return -EINVAL;
 763         }
 764
 765         ret = cryp_setup_context(ctx, device_data);
 766
 767         if (ret)
 768                 goto out;
 769
 770         if (cryp_mode == CRYP_MODE_INTERRUPT) {
 771                 cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO |
 772                                     CRYP_IRQ_SRC_OUTPUT_FIFO);
 773
 774                 /*
 775                  * ctx->outlen is decremented in the cryp_interrupt_handler
 776                  * function. We had to add cpu_relax() (barrier) to make sure
 777                  * that gcc didn't optimze away this variable.
 778                  */
 779                 while (ctx->outlen > 0)
 780                         cpu_relax();
 781         } else if (cryp_mode == CRYP_MODE_POLLING ||
 782                    cryp_mode == CRYP_MODE_DMA) {
 783                 /*
 784                  * The reason for having DMA in this if case is that if we are
 785                  * running cryp_mode = 2, then we separate DMA routines for
 786                  * handling cipher/plaintext > blocksize, except when
 787                  * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use
 788                  * the polling mode. Overhead of doing DMA setup eats up the
 789                  * benefits using it.
 790                  */
 791                 cryp_polling_mode(ctx, device_data);
 792         } else {
 793                 dev_err(ctx->device->dev, "[%s]: Invalid operation mode!",
 794                         __func__);
 795                 ret = -EPERM;
 796                 goto out;
 797         }
 798
 799         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 800         ctx->updated = 1;
 801
 802 out:
 803         ctx->indata = indata;
 804         ctx->outdata = outdata;
 805         ctx->datalen = datalen;
 806         ctx->outlen = outlen;
 807
 808         return ret;
 809 }
 810
 811 static int get_nents(struct scatterlist *sg, int nbytes)
 812 {
 813         int nents = 0;
 814
 815         while (nbytes > 0) {
 816                 nbytes -= sg->length;
 817                 sg = sg_next(sg);
 818                 nents++;
 819         }
 820
 821         return nents;
 822 }
 823
 824 static int ablk_dma_crypt(struct skcipher_request *areq)
 825 {
 826         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
 827         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
 828         struct cryp_device_data *device_data;
 829
 830         int bytes_written = 0;
 831         int bytes_read = 0;
 832         int ret;
 833
 834         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 835
 836         ctx->datalen = areq->cryptlen;
 837         ctx->outlen = areq->cryptlen;
 838
 839         ret = cryp_get_device_data(ctx, &device_data);
 840         if (ret)
 841                 return ret;
 842
 843         ret = cryp_setup_context(ctx, device_data);
 844         if (ret)
 845                 goto out;
 846
 847         /* We have the device now, so store the nents in the dma struct. */
 848         ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen);
 849         ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen);
 850
 851         /* Enable DMA in- and output. */
 852         cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS);
 853
 854         bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen);
 855         bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written);
 856
 857         wait_for_completion(&ctx->device->dma.cryp_dma_complete);
 858         cryp_dma_done(ctx);
 859
 860         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 861         ctx->updated = 1;
 862
 863 out:
 864         spin_lock(&device_data->ctx_lock);
 865         device_data->current_ctx = NULL;
 866         ctx->device = NULL;
 867         spin_unlock(&device_data->ctx_lock);
 868
 869         /*
 870          * The down_interruptible part for this semaphore is called in
 871          * cryp_get_device_data.
 872          */
 873         up(&driver_data.device_allocation);
 874
 875         if (unlikely(bytes_written != bytes_read))
 876                 return -EPERM;
 877
 878         return 0;
 879 }
 880
 881 static int ablk_crypt(struct skcipher_request *areq)
 882 {
 883         struct skcipher_walk walk;
 884         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
 885         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
 886         struct cryp_device_data *device_data;
 887         unsigned long src_paddr;
 888         unsigned long dst_paddr;
 889         int ret;
 890         int nbytes;
 891
 892         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 893
 894         ret = cryp_get_device_data(ctx, &device_data);
 895         if (ret)
 896                 goto out;
 897
 898         ret = skcipher_walk_async(&walk, areq);
 899
 900         if (ret) {
 901                 pr_err(DEV_DBG_NAME "[%s]: skcipher_walk_async() failed!",
 902                         __func__);
 903                 goto out;
 904         }
 905
 906         while ((nbytes = walk.nbytes) > 0) {
 907                 ctx->iv = walk.iv;
 908                 src_paddr = (page_to_phys(walk.src.phys.page) + walk.src.phys.offset);
 909                 ctx->indata = phys_to_virt(src_paddr);
 910
 911                 dst_paddr = (page_to_phys(walk.dst.phys.page) + walk.dst.phys.offset);
 912                 ctx->outdata = phys_to_virt(dst_paddr);
 913
 914                 ctx->datalen = nbytes - (nbytes % ctx->blocksize);
 915
 916                 ret = hw_crypt_noxts(ctx, device_data);
 917                 if (ret)
 918                         goto out;
 919
 920                 nbytes -= ctx->datalen;
 921                 ret = skcipher_walk_done(&walk, nbytes);
 922                 if (ret)
 923                         goto out;
 924         }
 925
 926 out:
 927         /* Release the device */
 928         spin_lock(&device_data->ctx_lock);
 929         device_data->current_ctx = NULL;
 930         ctx->device = NULL;
 931         spin_unlock(&device_data->ctx_lock);
 932
 933         /*
 934          * The down_interruptible part for this semaphore is called in
 935          * cryp_get_device_data.
 936          */
 937         up(&driver_data.device_allocation);
 938
 939         return ret;
 940 }
 941
 942 static int aes_skcipher_setkey(struct crypto_skcipher *cipher,
 943                                  const u8 *key, unsigned int keylen)
 944 {
 945         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
 946
 947         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 948
 949         switch (keylen) {
 950         case AES_KEYSIZE_128:
 951                 ctx->config.keysize = CRYP_KEY_SIZE_128;
 952                 break;
 953
 954         case AES_KEYSIZE_192:
 955                 ctx->config.keysize = CRYP_KEY_SIZE_192;
 956                 break;
 957
 958         case AES_KEYSIZE_256:
 959                 ctx->config.keysize = CRYP_KEY_SIZE_256;
 960                 break;
 961
 962         default:
 963                 pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__);
 964                 return -EINVAL;
 965         }
 966
 967         memcpy(ctx->key, key, keylen);
 968         ctx->keylen = keylen;
 969
 970         ctx->updated = 0;
 971
 972         return 0;
 973 }
 974
 975 static int des_skcipher_setkey(struct crypto_skcipher *cipher,
 976                                  const u8 *key, unsigned int keylen)
 977 {
 978         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
 979         int err;
 980
 981         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 982
 983         err = verify_skcipher_des_key(cipher, key);
 984         if (err)
 985                 return err;
 986
 987         memcpy(ctx->key, key, keylen);
 988         ctx->keylen = keylen;
 989
 990         ctx->updated = 0;
 991         return 0;
 992 }
 993
 994 static int des3_skcipher_setkey(struct crypto_skcipher *cipher,
 995                                   const u8 *key, unsigned int keylen)
 996 {
 997         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
 998         int err;
 999
1000         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1001
1002         err = verify_skcipher_des3_key(cipher, key);
1003         if (err)
1004                 return err;
1005
1006         memcpy(ctx->key, key, keylen);
1007         ctx->keylen = keylen;
1008
1009         ctx->updated = 0;
1010         return 0;
1011 }
1012
1013 static int cryp_blk_encrypt(struct skcipher_request *areq)
1014 {
1015         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1016         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
1017
1018         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1019
1020         ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT;
1021
1022         /*
1023          * DMA does not work for DES due to a hw bug */
1024         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1025                 return ablk_dma_crypt(areq);
1026
1027         /* For everything except DMA, we run the non DMA version. */
1028         return ablk_crypt(areq);
1029 }
1030
1031 static int cryp_blk_decrypt(struct skcipher_request *areq)
1032 {
1033         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1034         struct cryp_ctx *ctx = crypto_skcipher_ctx(cipher);
1035
1036         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1037
1038         ctx->config.algodir = CRYP_ALGORITHM_DECRYPT;
1039
1040         /* DMA does not work for DES due to a hw bug */
1041         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1042                 return ablk_dma_crypt(areq);
1043
1044         /* For everything except DMA, we run the non DMA version. */
1045         return ablk_crypt(areq);
1046 }
1047
1048 struct cryp_algo_template {
1049         enum cryp_algo_mode algomode;
1050         struct skcipher_alg skcipher;
1051 };
1052
1053 static int cryp_init_tfm(struct crypto_skcipher *tfm)
1054 {
1055         struct cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
1056         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1057         struct cryp_algo_template *cryp_alg = container_of(alg,
1058                         struct cryp_algo_template,
1059                         skcipher);
1060
1061         ctx->config.algomode = cryp_alg->algomode;
1062         ctx->blocksize = crypto_skcipher_blocksize(tfm);
1063
1064         return 0;
1065 }
1066
1067 static struct cryp_algo_template cryp_algs[] = {
1068         {
1069                 .algomode = CRYP_ALGO_AES_ECB,
1070                 .skcipher = {
1071                         .base.cra_name          = "ecb(aes)",
1072                         .base.cra_driver_name   = "ecb-aes-ux500",
1073                         .base.cra_priority      = 300,
1074                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1075                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1076                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1077                         .base.cra_alignmask     = 3,
1078                         .base.cra_module        = THIS_MODULE,
1079
1080                         .min_keysize            = AES_MIN_KEY_SIZE,
1081                         .max_keysize            = AES_MAX_KEY_SIZE,
1082                         .setkey                 = aes_skcipher_setkey,
1083                         .encrypt                = cryp_blk_encrypt,
1084                         .decrypt                = cryp_blk_decrypt,
1085                         .init                   = cryp_init_tfm,
1086                 }
1087         },
1088         {
1089                 .algomode = CRYP_ALGO_AES_CBC,
1090                 .skcipher = {
1091                         .base.cra_name          = "cbc(aes)",
1092                         .base.cra_driver_name   = "cbc-aes-ux500",
1093                         .base.cra_priority      = 300,
1094                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1095                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1096                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1097                         .base.cra_alignmask     = 3,
1098                         .base.cra_module        = THIS_MODULE,
1099
1100                         .min_keysize            = AES_MIN_KEY_SIZE,
1101                         .max_keysize            = AES_MAX_KEY_SIZE,
1102                         .setkey                 = aes_skcipher_setkey,
1103                         .encrypt                = cryp_blk_encrypt,
1104                         .decrypt                = cryp_blk_decrypt,
1105                         .init                   = cryp_init_tfm,
1106                         .ivsize                 = AES_BLOCK_SIZE,
1107                 }
1108         },
1109         {
1110                 .algomode = CRYP_ALGO_AES_CTR,
1111                 .skcipher = {
1112                         .base.cra_name          = "ctr(aes)",
1113                         .base.cra_driver_name   = "ctr-aes-ux500",
1114                         .base.cra_priority      = 300,
1115                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1116                         .base.cra_blocksize     = 1,
1117                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1118                         .base.cra_alignmask     = 3,
1119                         .base.cra_module        = THIS_MODULE,
1120
1121                         .min_keysize            = AES_MIN_KEY_SIZE,
1122                         .max_keysize            = AES_MAX_KEY_SIZE,
1123                         .setkey                 = aes_skcipher_setkey,
1124                         .encrypt                = cryp_blk_encrypt,
1125                         .decrypt                = cryp_blk_decrypt,
1126                         .init                   = cryp_init_tfm,
1127                         .ivsize                 = AES_BLOCK_SIZE,
1128                         .chunksize              = AES_BLOCK_SIZE,
1129                 }
1130         },
1131         {
1132                 .algomode = CRYP_ALGO_DES_ECB,
1133                 .skcipher = {
1134                         .base.cra_name          = "ecb(des)",
1135                         .base.cra_driver_name   = "ecb-des-ux500",
1136                         .base.cra_priority      = 300,
1137                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1138                         .base.cra_blocksize     = DES_BLOCK_SIZE,
1139                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1140                         .base.cra_alignmask     = 3,
1141                         .base.cra_module        = THIS_MODULE,
1142
1143                         .min_keysize            = DES_KEY_SIZE,
1144                         .max_keysize            = DES_KEY_SIZE,
1145                         .setkey                 = des_skcipher_setkey,
1146                         .encrypt                = cryp_blk_encrypt,
1147                         .decrypt                = cryp_blk_decrypt,
1148                         .init                   = cryp_init_tfm,
1149                 }
1150         },
1151         {
1152                 .algomode = CRYP_ALGO_TDES_ECB,
1153                 .skcipher = {
1154                         .base.cra_name          = "ecb(des3_ede)",
1155                         .base.cra_driver_name   = "ecb-des3_ede-ux500",
1156                         .base.cra_priority      = 300,
1157                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1158                         .base.cra_blocksize     = DES3_EDE_BLOCK_SIZE,
1159                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1160                         .base.cra_alignmask     = 3,
1161                         .base.cra_module        = THIS_MODULE,
1162
1163                         .min_keysize            = DES3_EDE_KEY_SIZE,
1164                         .max_keysize            = DES3_EDE_KEY_SIZE,
1165                         .setkey                 = des3_skcipher_setkey,
1166                         .encrypt                = cryp_blk_encrypt,
1167                         .decrypt                = cryp_blk_decrypt,
1168                         .init                   = cryp_init_tfm,
1169                 }
1170         },
1171         {
1172                 .algomode = CRYP_ALGO_DES_CBC,
1173                 .skcipher = {
1174                         .base.cra_name          = "cbc(des)",
1175                         .base.cra_driver_name   = "cbc-des-ux500",
1176                         .base.cra_priority      = 300,
1177                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1178                         .base.cra_blocksize     = DES_BLOCK_SIZE,
1179                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1180                         .base.cra_alignmask     = 3,
1181                         .base.cra_module        = THIS_MODULE,
1182
1183                         .min_keysize            = DES_KEY_SIZE,
1184                         .max_keysize            = DES_KEY_SIZE,
1185                         .setkey                 = des_skcipher_setkey,
1186                         .encrypt                = cryp_blk_encrypt,
1187                         .decrypt                = cryp_blk_decrypt,
1188                         .ivsize                 = DES_BLOCK_SIZE,
1189                         .init                   = cryp_init_tfm,
1190                 }
1191         },
1192         {
1193                 .algomode = CRYP_ALGO_TDES_CBC,
1194                 .skcipher = {
1195                         .base.cra_name          = "cbc(des3_ede)",
1196                         .base.cra_driver_name   = "cbc-des3_ede-ux500",
1197                         .base.cra_priority      = 300,
1198                         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1199                         .base.cra_blocksize     = DES3_EDE_BLOCK_SIZE,
1200                         .base.cra_ctxsize       = sizeof(struct cryp_ctx),
1201                         .base.cra_alignmask     = 3,
1202                         .base.cra_module        = THIS_MODULE,
1203
1204                         .min_keysize            = DES3_EDE_KEY_SIZE,
1205                         .max_keysize            = DES3_EDE_KEY_SIZE,
1206                         .setkey                 = des3_skcipher_setkey,
1207                         .encrypt                = cryp_blk_encrypt,
1208                         .decrypt                = cryp_blk_decrypt,
1209                         .ivsize                 = DES3_EDE_BLOCK_SIZE,
1210                         .init                   = cryp_init_tfm,
1211                 }
1212         }
1213 };
1214
1215 /**
1216  * cryp_algs_register_all -
1217  */
1218 static int cryp_algs_register_all(void)
1219 {
1220         int ret;
1221         int i;
1222         int count;
1223
1224         pr_debug("[%s]", __func__);
1225
1226         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) {
1227                 ret = crypto_register_skcipher(&cryp_algs[i].skcipher);
1228                 if (ret) {
1229                         count = i;
1230                         pr_err("[%s] alg registration failed",
1231                                         cryp_algs[i].skcipher.base.cra_driver_name);
1232                         goto unreg;
1233                 }
1234         }
1235         return 0;
1236 unreg:
1237         for (i = 0; i < count; i++)
1238                 crypto_unregister_skcipher(&cryp_algs[i].skcipher);
1239         return ret;
1240 }
1241
1242 /**
1243  * cryp_algs_unregister_all -
1244  */
1245 static void cryp_algs_unregister_all(void)
1246 {
1247         int i;
1248
1249         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1250
1251         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++)
1252                 crypto_unregister_skcipher(&cryp_algs[i].skcipher);
1253 }
1254
1255 static int ux500_cryp_probe(struct platform_device *pdev)
1256 {
1257         int ret;
1258         struct resource *res;
1259         struct resource *res_irq;
1260         struct cryp_device_data *device_data;
1261         struct cryp_protection_config prot = {
1262                 .privilege_access = CRYP_STATE_ENABLE
1263         };
1264         struct device *dev = &pdev->dev;
1265
1266         dev_dbg(dev, "[%s]", __func__);
1267         device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC);
1268         if (!device_data) {
1269                 ret = -ENOMEM;
1270                 goto out;
1271         }
1272
1273         device_data->dev = dev;
1274         device_data->current_ctx = NULL;
1275
1276         /* Grab the DMA configuration from platform data. */
1277         mem_to_engine = &((struct cryp_platform_data *)
1278                          dev->platform_data)->mem_to_engine;
1279         engine_to_mem = &((struct cryp_platform_data *)
1280                          dev->platform_data)->engine_to_mem;
1281
1282         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1283         if (!res) {
1284                 dev_err(dev, "[%s]: platform_get_resource() failed",
1285                                 __func__);
1286                 ret = -ENODEV;
1287                 goto out;
1288         }
1289
1290         device_data->phybase = res->start;
1291         device_data->base = devm_ioremap_resource(dev, res);
1292         if (IS_ERR(device_data->base)) {
1293                 dev_err(dev, "[%s]: ioremap failed!", __func__);
1294                 ret = PTR_ERR(device_data->base);
1295                 goto out;
1296         }
1297
1298         spin_lock_init(&device_data->ctx_lock);
1299         spin_lock_init(&device_data->power_state_spinlock);
1300
1301         /* Enable power for CRYP hardware block */
1302         device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape");
1303         if (IS_ERR(device_data->pwr_regulator)) {
1304                 dev_err(dev, "[%s]: could not get cryp regulator", __func__);
1305                 ret = PTR_ERR(device_data->pwr_regulator);
1306                 device_data->pwr_regulator = NULL;
1307                 goto out;
1308         }
1309
1310         /* Enable the clk for CRYP hardware block */
1311         device_data->clk = devm_clk_get(&pdev->dev, NULL);
1312         if (IS_ERR(device_data->clk)) {
1313                 dev_err(dev, "[%s]: clk_get() failed!", __func__);
1314                 ret = PTR_ERR(device_data->clk);
1315                 goto out_regulator;
1316         }
1317
1318         ret = clk_prepare(device_data->clk);
1319         if (ret) {
1320                 dev_err(dev, "[%s]: clk_prepare() failed!", __func__);
1321                 goto out_regulator;
1322         }
1323
1324         /* Enable device power (and clock) */
1325         ret = cryp_enable_power(device_data->dev, device_data, false);
1326         if (ret) {
1327                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1328                 goto out_clk_unprepare;
1329         }
1330
1331         if (cryp_check(device_data)) {
1332                 dev_err(dev, "[%s]: cryp_check() failed!", __func__);
1333                 ret = -EINVAL;
1334                 goto out_power;
1335         }
1336
1337         if (cryp_configure_protection(device_data, &prot)) {
1338                 dev_err(dev, "[%s]: cryp_configure_protection() failed!",
1339                         __func__);
1340                 ret = -EINVAL;
1341                 goto out_power;
1342         }
1343
1344         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1345         if (!res_irq) {
1346                 dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
1347                         __func__);
1348                 ret = -ENODEV;
1349                 goto out_power;
1350         }
1351
1352         ret = devm_request_irq(&pdev->dev, res_irq->start,
1353                                cryp_interrupt_handler, 0, "cryp1", device_data);
1354         if (ret) {
1355                 dev_err(dev, "[%s]: Unable to request IRQ", __func__);
1356                 goto out_power;
1357         }
1358
1359         if (cryp_mode == CRYP_MODE_DMA)
1360                 cryp_dma_setup_channel(device_data, dev);
1361
1362         platform_set_drvdata(pdev, device_data);
1363
1364         /* Put the new device into the device list... */
1365         klist_add_tail(&device_data->list_node, &driver_data.device_list);
1366
1367         /* ... and signal that a new device is available. */
1368         up(&driver_data.device_allocation);
1369
1370         atomic_set(&session_id, 1);
1371
1372         ret = cryp_algs_register_all();
1373         if (ret) {
1374                 dev_err(dev, "[%s]: cryp_algs_register_all() failed!",
1375                         __func__);
1376                 goto out_power;
1377         }
1378
1379         dev_info(dev, "successfully registered\n");
1380
1381         return 0;
1382
1383 out_power:
1384         cryp_disable_power(device_data->dev, device_data, false);
1385
1386 out_clk_unprepare:
1387         clk_unprepare(device_data->clk);
1388
1389 out_regulator:
1390         regulator_put(device_data->pwr_regulator);
1391
1392 out:
1393         return ret;
1394 }
1395
1396 static int ux500_cryp_remove(struct platform_device *pdev)
1397 {
1398         struct cryp_device_data *device_data;
1399
1400         dev_dbg(&pdev->dev, "[%s]", __func__);
1401         device_data = platform_get_drvdata(pdev);
1402         if (!device_data) {
1403                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1404                         __func__);
1405                 return -ENOMEM;
1406         }
1407
1408         /* Try to decrease the number of available devices. */
1409         if (down_trylock(&driver_data.device_allocation))
1410                 return -EBUSY;
1411
1412         /* Check that the device is free */
1413         spin_lock(&device_data->ctx_lock);
1414         /* current_ctx allocates a device, NULL = unallocated */
1415         if (device_data->current_ctx) {
1416                 /* The device is busy */
1417                 spin_unlock(&device_data->ctx_lock);
1418                 /* Return the device to the pool. */
1419                 up(&driver_data.device_allocation);
1420                 return -EBUSY;
1421         }
1422
1423         spin_unlock(&device_data->ctx_lock);
1424
1425         /* Remove the device from the list */
1426         if (klist_node_attached(&device_data->list_node))
1427                 klist_remove(&device_data->list_node);
1428
1429         /* If this was the last device, remove the services */
1430         if (list_empty(&driver_data.device_list.k_list))
1431                 cryp_algs_unregister_all();
1432
1433         if (cryp_disable_power(&pdev->dev, device_data, false))
1434                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1435                         __func__);
1436
1437         clk_unprepare(device_data->clk);
1438         regulator_put(device_data->pwr_regulator);
1439
1440         return 0;
1441 }
1442
1443 static void ux500_cryp_shutdown(struct platform_device *pdev)
1444 {
1445         struct cryp_device_data *device_data;
1446
1447         dev_dbg(&pdev->dev, "[%s]", __func__);
1448
1449         device_data = platform_get_drvdata(pdev);
1450         if (!device_data) {
1451                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1452                         __func__);
1453                 return;
1454         }
1455
1456         /* Check that the device is free */
1457         spin_lock(&device_data->ctx_lock);
1458         /* current_ctx allocates a device, NULL = unallocated */
1459         if (!device_data->current_ctx) {
1460                 if (down_trylock(&driver_data.device_allocation))
1461                         dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
1462                                 "Shutting down anyway...", __func__);
1463                 /**
1464                  * (Allocate the device)
1465                  * Need to set this to non-null (dummy) value,
1466                  * to avoid usage if context switching.
1467                  */
1468                 device_data->current_ctx++;
1469         }
1470         spin_unlock(&device_data->ctx_lock);
1471
1472         /* Remove the device from the list */
1473         if (klist_node_attached(&device_data->list_node))
1474                 klist_remove(&device_data->list_node);
1475
1476         /* If this was the last device, remove the services */
1477         if (list_empty(&driver_data.device_list.k_list))
1478                 cryp_algs_unregister_all();
1479
1480         if (cryp_disable_power(&pdev->dev, device_data, false))
1481                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1482                         __func__);
1483
1484 }
1485
1486 #ifdef CONFIG_PM_SLEEP
1487 static int ux500_cryp_suspend(struct device *dev)
1488 {
1489         int ret;
1490         struct platform_device *pdev = to_platform_device(dev);
1491         struct cryp_device_data *device_data;
1492         struct resource *res_irq;
1493         struct cryp_ctx *temp_ctx = NULL;
1494
1495         dev_dbg(dev, "[%s]", __func__);
1496
1497         /* Handle state? */
1498         device_data = platform_get_drvdata(pdev);
1499         if (!device_data) {
1500                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1501                 return -ENOMEM;
1502         }
1503
1504         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1505         if (!res_irq)
1506                 dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
1507         else
1508                 disable_irq(res_irq->start);
1509
1510         spin_lock(&device_data->ctx_lock);
1511         if (!device_data->current_ctx)
1512                 device_data->current_ctx++;
1513         spin_unlock(&device_data->ctx_lock);
1514
1515         if (device_data->current_ctx == ++temp_ctx) {
1516                 if (down_interruptible(&driver_data.device_allocation))
1517                         dev_dbg(dev, "[%s]: down_interruptible() failed",
1518                                 __func__);
1519                 ret = cryp_disable_power(dev, device_data, false);
1520
1521         } else
1522                 ret = cryp_disable_power(dev, device_data, true);
1523
1524         if (ret)
1525                 dev_err(dev, "[%s]: cryp_disable_power()", __func__);
1526
1527         return ret;
1528 }
1529
1530 static int ux500_cryp_resume(struct device *dev)
1531 {
1532         int ret = 0;
1533         struct platform_device *pdev = to_platform_device(dev);
1534         struct cryp_device_data *device_data;
1535         struct resource *res_irq;
1536         struct cryp_ctx *temp_ctx = NULL;
1537
1538         dev_dbg(dev, "[%s]", __func__);
1539
1540         device_data = platform_get_drvdata(pdev);
1541         if (!device_data) {
1542                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1543                 return -ENOMEM;
1544         }
1545
1546         spin_lock(&device_data->ctx_lock);
1547         if (device_data->current_ctx == ++temp_ctx)
1548                 device_data->current_ctx = NULL;
1549         spin_unlock(&device_data->ctx_lock);
1550
1551
1552         if (!device_data->current_ctx)
1553                 up(&driver_data.device_allocation);
1554         else
1555                 ret = cryp_enable_power(dev, device_data, true);
1556
1557         if (ret)
1558                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1559         else {
1560                 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1561                 if (res_irq)
1562                         enable_irq(res_irq->start);
1563         }
1564
1565         return ret;
1566 }
1567 #endif
1568
1569 static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
1570
1571 static const struct of_device_id ux500_cryp_match[] = {
1572         { .compatible = "stericsson,ux500-cryp" },
1573         { },
1574 };
1575 MODULE_DEVICE_TABLE(of, ux500_cryp_match);
1576
1577 static struct platform_driver cryp_driver = {
1578         .probe  = ux500_cryp_probe,
1579         .remove = ux500_cryp_remove,
1580         .shutdown = ux500_cryp_shutdown,
1581         .driver = {
1582                 .name  = "cryp1",
1583                 .of_match_table = ux500_cryp_match,
1584                 .pm    = &ux500_cryp_pm,
1585         }
1586 };
1587
1588 static int __init ux500_cryp_mod_init(void)
1589 {
1590         pr_debug("[%s] is called!", __func__);
1591         klist_init(&driver_data.device_list, NULL, NULL);
1592         /* Initialize the semaphore to 0 devices (locked state) */
1593         sema_init(&driver_data.device_allocation, 0);
1594         return platform_driver_register(&cryp_driver);
1595 }
1596
1597 static void __exit ux500_cryp_mod_fini(void)
1598 {
1599         pr_debug("[%s] is called!", __func__);
1600         platform_driver_unregister(&cryp_driver);
1601 }
1602
1603 module_init(ux500_cryp_mod_init);
1604 module_exit(ux500_cryp_mod_fini);
1605
1606 module_param(cryp_mode, int, 0);
1607
1608 MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine.");
1609 MODULE_ALIAS_CRYPTO("aes-all");
1610 MODULE_ALIAS_CRYPTO("des-all");
1611
1612 MODULE_LICENSE("GPL");