drivers/crypto/ccp/ccp-ops.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8  * Author: Gary R Hook <gary.hook@amd.com>
   9  */
  10
  11 #include <linux/dma-mapping.h>
  12 #include <linux/module.h>
  13 #include <linux/kernel.h>
  14 #include <linux/interrupt.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/des.h>
  17 #include <linux/ccp.h>
  18
  19 #include "ccp-dev.h"
  20
  21 /* SHA initial context values */
  22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25         cpu_to_be32(SHA1_H4),
  26 };
  27
  28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  29         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  30         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  31         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  32         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  33 };
  34
  35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  36         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  37         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  38         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  39         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  40 };
  41
  42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  43         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  44         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  45         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  46         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  47 };
  48
  49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  50         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  51         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  52         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  53         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  54 };
  55
  56 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  57                                         ccp_gen_jobid(ccp) : 0)
  58
  59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  60 {
  61         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  62 }
  63
  64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  65 {
  66         if (wa->dma_count)
  67                 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  68
  69         wa->dma_count = 0;
  70 }
  71
  72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  73                                 struct scatterlist *sg, u64 len,
  74                                 enum dma_data_direction dma_dir)
  75 {
  76         memset(wa, 0, sizeof(*wa));
  77
  78         wa->sg = sg;
  79         if (!sg)
  80                 return 0;
  81
  82         wa->nents = sg_nents_for_len(sg, len);
  83         if (wa->nents < 0)
  84                 return wa->nents;
  85
  86         wa->bytes_left = len;
  87         wa->sg_used = 0;
  88
  89         if (len == 0)
  90                 return 0;
  91
  92         if (dma_dir == DMA_NONE)
  93                 return 0;
  94
  95         wa->dma_sg = sg;
  96         wa->dma_sg_head = sg;
  97         wa->dma_dev = dev;
  98         wa->dma_dir = dma_dir;
  99         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 100         if (!wa->dma_count)
 101                 return -ENOMEM;
 102
 103         return 0;
 104 }
 105
 106 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 107 {
 108         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 109         unsigned int sg_combined_len = 0;
 110
 111         if (!wa->sg)
 112                 return;
 113
 114         wa->sg_used += nbytes;
 115         wa->bytes_left -= nbytes;
 116         if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
 117                 /* Advance to the next DMA scatterlist entry */
 118                 wa->dma_sg = sg_next(wa->dma_sg);
 119
 120                 /* In the case that the DMA mapped scatterlist has entries
 121                  * that have been merged, the non-DMA mapped scatterlist
 122                  * must be advanced multiple times for each merged entry.
 123                  * This ensures that the current non-DMA mapped entry
 124                  * corresponds to the current DMA mapped entry.
 125                  */
 126                 do {
 127                         sg_combined_len += wa->sg->length;
 128                         wa->sg = sg_next(wa->sg);
 129                 } while (wa->sg_used > sg_combined_len);
 130
 131                 wa->sg_used = 0;
 132         }
 133 }
 134
 135 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 136 {
 137         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 138                 if (wa->address)
 139                         dma_pool_free(wa->dma_pool, wa->address,
 140                                       wa->dma.address);
 141         } else {
 142                 if (wa->dma.address)
 143                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 144                                          wa->dma.dir);
 145                 kfree(wa->address);
 146         }
 147
 148         wa->address = NULL;
 149         wa->dma.address = 0;
 150 }
 151
 152 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 153                                 struct ccp_cmd_queue *cmd_q,
 154                                 unsigned int len,
 155                                 enum dma_data_direction dir)
 156 {
 157         memset(wa, 0, sizeof(*wa));
 158
 159         if (!len)
 160                 return 0;
 161
 162         wa->dev = cmd_q->ccp->dev;
 163         wa->length = len;
 164
 165         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 166                 wa->dma_pool = cmd_q->dma_pool;
 167
 168                 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
 169                                              &wa->dma.address);
 170                 if (!wa->address)
 171                         return -ENOMEM;
 172
 173                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 174
 175         } else {
 176                 wa->address = kzalloc(len, GFP_KERNEL);
 177                 if (!wa->address)
 178                         return -ENOMEM;
 179
 180                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 181                                                  dir);
 182                 if (dma_mapping_error(wa->dev, wa->dma.address)) {
 183                         kfree(wa->address);
 184                         wa->address = NULL;
 185                         return -ENOMEM;
 186                 }
 187
 188                 wa->dma.length = len;
 189         }
 190         wa->dma.dir = dir;
 191
 192         return 0;
 193 }
 194
 195 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 196                            struct scatterlist *sg, unsigned int sg_offset,
 197                            unsigned int len)
 198 {
 199         WARN_ON(!wa->address);
 200
 201         if (len > (wa->length - wa_offset))
 202                 return -EINVAL;
 203
 204         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 205                                  0);
 206         return 0;
 207 }
 208
 209 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 210                             struct scatterlist *sg, unsigned int sg_offset,
 211                             unsigned int len)
 212 {
 213         WARN_ON(!wa->address);
 214
 215         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 216                                  1);
 217 }
 218
 219 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 220                                    unsigned int wa_offset,
 221                                    struct scatterlist *sg,
 222                                    unsigned int sg_offset,
 223                                    unsigned int len)
 224 {
 225         u8 *p, *q;
 226         int     rc;
 227
 228         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 229         if (rc)
 230                 return rc;
 231
 232         p = wa->address + wa_offset;
 233         q = p + len - 1;
 234         while (p < q) {
 235                 *p = *p ^ *q;
 236                 *q = *p ^ *q;
 237                 *p = *p ^ *q;
 238                 p++;
 239                 q--;
 240         }
 241         return 0;
 242 }
 243
 244 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 245                                     unsigned int wa_offset,
 246                                     struct scatterlist *sg,
 247                                     unsigned int sg_offset,
 248                                     unsigned int len)
 249 {
 250         u8 *p, *q;
 251
 252         p = wa->address + wa_offset;
 253         q = p + len - 1;
 254         while (p < q) {
 255                 *p = *p ^ *q;
 256                 *q = *p ^ *q;
 257                 *p = *p ^ *q;
 258                 p++;
 259                 q--;
 260         }
 261
 262         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 263 }
 264
 265 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 266 {
 267         ccp_dm_free(&data->dm_wa);
 268         ccp_sg_free(&data->sg_wa);
 269 }
 270
 271 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 272                          struct scatterlist *sg, u64 sg_len,
 273                          unsigned int dm_len,
 274                          enum dma_data_direction dir)
 275 {
 276         int ret;
 277
 278         memset(data, 0, sizeof(*data));
 279
 280         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 281                                    dir);
 282         if (ret)
 283                 goto e_err;
 284
 285         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 286         if (ret)
 287                 goto e_err;
 288
 289         return 0;
 290
 291 e_err:
 292         ccp_free_data(data, cmd_q);
 293
 294         return ret;
 295 }
 296
 297 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 298 {
 299         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 300         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 301         unsigned int buf_count, nbytes;
 302
 303         /* Clear the buffer if setting it */
 304         if (!from)
 305                 memset(dm_wa->address, 0, dm_wa->length);
 306
 307         if (!sg_wa->sg)
 308                 return 0;
 309
 310         /* Perform the copy operation
 311          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 312          *   an unsigned int
 313          */
 314         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 315         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 316                                  nbytes, from);
 317
 318         /* Update the structures and generate the count */
 319         buf_count = 0;
 320         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 321                 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
 322                              dm_wa->length - buf_count);
 323                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 324
 325                 buf_count += nbytes;
 326                 ccp_update_sg_workarea(sg_wa, nbytes);
 327         }
 328
 329         return buf_count;
 330 }
 331
 332 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 333 {
 334         return ccp_queue_buf(data, 0);
 335 }
 336
 337 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 338 {
 339         return ccp_queue_buf(data, 1);
 340 }
 341
 342 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 343                              struct ccp_op *op, unsigned int block_size,
 344                              bool blocksize_op)
 345 {
 346         unsigned int sg_src_len, sg_dst_len, op_len;
 347
 348         /* The CCP can only DMA from/to one address each per operation. This
 349          * requires that we find the smallest DMA area between the source
 350          * and destination. The resulting len values will always be <= UINT_MAX
 351          * because the dma length is an unsigned int.
 352          */
 353         sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
 354         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 355
 356         if (dst) {
 357                 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
 358                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 359                 op_len = min(sg_src_len, sg_dst_len);
 360         } else {
 361                 op_len = sg_src_len;
 362         }
 363
 364         /* The data operation length will be at least block_size in length
 365          * or the smaller of available sg room remaining for the source or
 366          * the destination
 367          */
 368         op_len = max(op_len, block_size);
 369
 370         /* Unless we have to buffer data, there's no reason to wait */
 371         op->soc = 0;
 372
 373         if (sg_src_len < block_size) {
 374                 /* Not enough data in the sg element, so it
 375                  * needs to be buffered into a blocksize chunk
 376                  */
 377                 int cp_len = ccp_fill_queue_buf(src);
 378
 379                 op->soc = 1;
 380                 op->src.u.dma.address = src->dm_wa.dma.address;
 381                 op->src.u.dma.offset = 0;
 382                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 383         } else {
 384                 /* Enough data in the sg element, but we need to
 385                  * adjust for any previously copied data
 386                  */
 387                 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
 388                 op->src.u.dma.offset = src->sg_wa.sg_used;
 389                 op->src.u.dma.length = op_len & ~(block_size - 1);
 390
 391                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 392         }
 393
 394         if (dst) {
 395                 if (sg_dst_len < block_size) {
 396                         /* Not enough room in the sg element or we're on the
 397                          * last piece of data (when using padding), so the
 398                          * output needs to be buffered into a blocksize chunk
 399                          */
 400                         op->soc = 1;
 401                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 402                         op->dst.u.dma.offset = 0;
 403                         op->dst.u.dma.length = op->src.u.dma.length;
 404                 } else {
 405                         /* Enough room in the sg element, but we need to
 406                          * adjust for any previously used area
 407                          */
 408                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
 409                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 410                         op->dst.u.dma.length = op->src.u.dma.length;
 411                 }
 412         }
 413 }
 414
 415 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 416                              struct ccp_op *op)
 417 {
 418         op->init = 0;
 419
 420         if (dst) {
 421                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 422                         ccp_empty_queue_buf(dst);
 423                 else
 424                         ccp_update_sg_workarea(&dst->sg_wa,
 425                                                op->dst.u.dma.length);
 426         }
 427 }
 428
 429 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 430                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 431                                u32 byte_swap, bool from)
 432 {
 433         struct ccp_op op;
 434
 435         memset(&op, 0, sizeof(op));
 436
 437         op.cmd_q = cmd_q;
 438         op.jobid = jobid;
 439         op.eom = 1;
 440
 441         if (from) {
 442                 op.soc = 1;
 443                 op.src.type = CCP_MEMTYPE_SB;
 444                 op.src.u.sb = sb;
 445                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 446                 op.dst.u.dma.address = wa->dma.address;
 447                 op.dst.u.dma.length = wa->length;
 448         } else {
 449                 op.src.type = CCP_MEMTYPE_SYSTEM;
 450                 op.src.u.dma.address = wa->dma.address;
 451                 op.src.u.dma.length = wa->length;
 452                 op.dst.type = CCP_MEMTYPE_SB;
 453                 op.dst.u.sb = sb;
 454         }
 455
 456         op.u.passthru.byte_swap = byte_swap;
 457
 458         return cmd_q->ccp->vdata->perform->passthru(&op);
 459 }
 460
 461 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 462                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 463                           u32 byte_swap)
 464 {
 465         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 466 }
 467
 468 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 469                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 470                             u32 byte_swap)
 471 {
 472         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 473 }
 474
 475 static noinline_for_stack int
 476 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 477 {
 478         struct ccp_aes_engine *aes = &cmd->u.aes;
 479         struct ccp_dm_workarea key, ctx;
 480         struct ccp_data src;
 481         struct ccp_op op;
 482         unsigned int dm_offset;
 483         int ret;
 484
 485         if (!((aes->key_len == AES_KEYSIZE_128) ||
 486               (aes->key_len == AES_KEYSIZE_192) ||
 487               (aes->key_len == AES_KEYSIZE_256)))
 488                 return -EINVAL;
 489
 490         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 491                 return -EINVAL;
 492
 493         if (aes->iv_len != AES_BLOCK_SIZE)
 494                 return -EINVAL;
 495
 496         if (!aes->key || !aes->iv || !aes->src)
 497                 return -EINVAL;
 498
 499         if (aes->cmac_final) {
 500                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 501                         return -EINVAL;
 502
 503                 if (!aes->cmac_key)
 504                         return -EINVAL;
 505         }
 506
 507         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 508         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 509
 510         ret = -EIO;
 511         memset(&op, 0, sizeof(op));
 512         op.cmd_q = cmd_q;
 513         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 514         op.sb_key = cmd_q->sb_key;
 515         op.sb_ctx = cmd_q->sb_ctx;
 516         op.init = 1;
 517         op.u.aes.type = aes->type;
 518         op.u.aes.mode = aes->mode;
 519         op.u.aes.action = aes->action;
 520
 521         /* All supported key sizes fit in a single (32-byte) SB entry
 522          * and must be in little endian format. Use the 256-bit byte
 523          * swap passthru option to convert from big endian to little
 524          * endian.
 525          */
 526         ret = ccp_init_dm_workarea(&key, cmd_q,
 527                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 528                                    DMA_TO_DEVICE);
 529         if (ret)
 530                 return ret;
 531
 532         dm_offset = CCP_SB_BYTES - aes->key_len;
 533         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 534         if (ret)
 535                 goto e_key;
 536         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 537                              CCP_PASSTHRU_BYTESWAP_256BIT);
 538         if (ret) {
 539                 cmd->engine_error = cmd_q->cmd_error;
 540                 goto e_key;
 541         }
 542
 543         /* The AES context fits in a single (32-byte) SB entry and
 544          * must be in little endian format. Use the 256-bit byte swap
 545          * passthru option to convert from big endian to little endian.
 546          */
 547         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 548                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 549                                    DMA_BIDIRECTIONAL);
 550         if (ret)
 551                 goto e_key;
 552
 553         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 554         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 555         if (ret)
 556                 goto e_ctx;
 557         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 558                              CCP_PASSTHRU_BYTESWAP_256BIT);
 559         if (ret) {
 560                 cmd->engine_error = cmd_q->cmd_error;
 561                 goto e_ctx;
 562         }
 563
 564         /* Send data to the CCP AES engine */
 565         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 566                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 567         if (ret)
 568                 goto e_ctx;
 569
 570         while (src.sg_wa.bytes_left) {
 571                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 572                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 573                         op.eom = 1;
 574
 575                         /* Push the K1/K2 key to the CCP now */
 576                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 577                                                op.sb_ctx,
 578                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 579                         if (ret) {
 580                                 cmd->engine_error = cmd_q->cmd_error;
 581                                 goto e_src;
 582                         }
 583
 584                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 585                                               aes->cmac_key_len);
 586                         if (ret)
 587                                 goto e_src;
 588                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 589                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 590                         if (ret) {
 591                                 cmd->engine_error = cmd_q->cmd_error;
 592                                 goto e_src;
 593                         }
 594                 }
 595
 596                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 597                 if (ret) {
 598                         cmd->engine_error = cmd_q->cmd_error;
 599                         goto e_src;
 600                 }
 601
 602                 ccp_process_data(&src, NULL, &op);
 603         }
 604
 605         /* Retrieve the AES context - convert from LE to BE using
 606          * 32-byte (256-bit) byteswapping
 607          */
 608         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 609                                CCP_PASSTHRU_BYTESWAP_256BIT);
 610         if (ret) {
 611                 cmd->engine_error = cmd_q->cmd_error;
 612                 goto e_src;
 613         }
 614
 615         /* ...but we only need AES_BLOCK_SIZE bytes */
 616         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 617         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 618
 619 e_src:
 620         ccp_free_data(&src, cmd_q);
 621
 622 e_ctx:
 623         ccp_dm_free(&ctx);
 624
 625 e_key:
 626         ccp_dm_free(&key);
 627
 628         return ret;
 629 }
 630
 631 static noinline_for_stack int
 632 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 633 {
 634         struct ccp_aes_engine *aes = &cmd->u.aes;
 635         struct ccp_dm_workarea key, ctx, final_wa, tag;
 636         struct ccp_data src, dst;
 637         struct ccp_data aad;
 638         struct ccp_op op;
 639         unsigned int dm_offset;
 640         unsigned int authsize;
 641         unsigned int jobid;
 642         unsigned int ilen;
 643         bool in_place = true; /* Default value */
 644         __be64 *final;
 645         int ret;
 646
 647         struct scatterlist *p_inp, sg_inp[2];
 648         struct scatterlist *p_tag, sg_tag[2];
 649         struct scatterlist *p_outp, sg_outp[2];
 650         struct scatterlist *p_aad;
 651
 652         if (!aes->iv)
 653                 return -EINVAL;
 654
 655         if (!((aes->key_len == AES_KEYSIZE_128) ||
 656                 (aes->key_len == AES_KEYSIZE_192) ||
 657                 (aes->key_len == AES_KEYSIZE_256)))
 658                 return -EINVAL;
 659
 660         if (!aes->key) /* Gotta have a key SGL */
 661                 return -EINVAL;
 662
 663         /* Zero defaults to 16 bytes, the maximum size */
 664         authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 665         switch (authsize) {
 666         case 16:
 667         case 15:
 668         case 14:
 669         case 13:
 670         case 12:
 671         case 8:
 672         case 4:
 673                 break;
 674         default:
 675                 return -EINVAL;
 676         }
 677
 678         /* First, decompose the source buffer into AAD & PT,
 679          * and the destination buffer into AAD, CT & tag, or
 680          * the input into CT & tag.
 681          * It is expected that the input and output SGs will
 682          * be valid, even if the AAD and input lengths are 0.
 683          */
 684         p_aad = aes->src;
 685         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 686         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 687         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 688                 ilen = aes->src_len;
 689                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 690         } else {
 691                 /* Input length for decryption includes tag */
 692                 ilen = aes->src_len - authsize;
 693                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 694         }
 695
 696         jobid = CCP_NEW_JOBID(cmd_q->ccp);
 697
 698         memset(&op, 0, sizeof(op));
 699         op.cmd_q = cmd_q;
 700         op.jobid = jobid;
 701         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 702         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 703         op.init = 1;
 704         op.u.aes.type = aes->type;
 705
 706         /* Copy the key to the LSB */
 707         ret = ccp_init_dm_workarea(&key, cmd_q,
 708                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 709                                    DMA_TO_DEVICE);
 710         if (ret)
 711                 return ret;
 712
 713         dm_offset = CCP_SB_BYTES - aes->key_len;
 714         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 715         if (ret)
 716                 goto e_key;
 717         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 718                              CCP_PASSTHRU_BYTESWAP_256BIT);
 719         if (ret) {
 720                 cmd->engine_error = cmd_q->cmd_error;
 721                 goto e_key;
 722         }
 723
 724         /* Copy the context (IV) to the LSB.
 725          * There is an assumption here that the IV is 96 bits in length, plus
 726          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 727          */
 728         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 729                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 730                                    DMA_BIDIRECTIONAL);
 731         if (ret)
 732                 goto e_key;
 733
 734         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 735         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 736         if (ret)
 737                 goto e_ctx;
 738
 739         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 740                              CCP_PASSTHRU_BYTESWAP_256BIT);
 741         if (ret) {
 742                 cmd->engine_error = cmd_q->cmd_error;
 743                 goto e_ctx;
 744         }
 745
 746         op.init = 1;
 747         if (aes->aad_len > 0) {
 748                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 749                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 750                                     AES_BLOCK_SIZE,
 751                                     DMA_TO_DEVICE);
 752                 if (ret)
 753                         goto e_ctx;
 754
 755                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 756                 op.u.aes.action = CCP_AES_GHASHAAD;
 757
 758                 while (aad.sg_wa.bytes_left) {
 759                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 760
 761                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 762                         if (ret) {
 763                                 cmd->engine_error = cmd_q->cmd_error;
 764                                 goto e_aad;
 765                         }
 766
 767                         ccp_process_data(&aad, NULL, &op);
 768                         op.init = 0;
 769                 }
 770         }
 771
 772         op.u.aes.mode = CCP_AES_MODE_GCTR;
 773         op.u.aes.action = aes->action;
 774
 775         if (ilen > 0) {
 776                 /* Step 2: Run a GCTR over the plaintext */
 777                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 778
 779                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 780                                     AES_BLOCK_SIZE,
 781                                     in_place ? DMA_BIDIRECTIONAL
 782                                              : DMA_TO_DEVICE);
 783                 if (ret)
 784                         goto e_aad;
 785
 786                 if (in_place) {
 787                         dst = src;
 788                 } else {
 789                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 790                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 791                         if (ret)
 792                                 goto e_src;
 793                 }
 794
 795                 op.soc = 0;
 796                 op.eom = 0;
 797                 op.init = 1;
 798                 while (src.sg_wa.bytes_left) {
 799                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 800                         if (!src.sg_wa.bytes_left) {
 801                                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 802
 803                                 if (nbytes) {
 804                                         op.eom = 1;
 805                                         op.u.aes.size = (nbytes * 8) - 1;
 806                                 }
 807                         }
 808
 809                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 810                         if (ret) {
 811                                 cmd->engine_error = cmd_q->cmd_error;
 812                                 goto e_dst;
 813                         }
 814
 815                         ccp_process_data(&src, &dst, &op);
 816                         op.init = 0;
 817                 }
 818         }
 819
 820         /* Step 3: Update the IV portion of the context with the original IV */
 821         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 822                                CCP_PASSTHRU_BYTESWAP_256BIT);
 823         if (ret) {
 824                 cmd->engine_error = cmd_q->cmd_error;
 825                 goto e_dst;
 826         }
 827
 828         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 829         if (ret)
 830                 goto e_dst;
 831
 832         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 833                              CCP_PASSTHRU_BYTESWAP_256BIT);
 834         if (ret) {
 835                 cmd->engine_error = cmd_q->cmd_error;
 836                 goto e_dst;
 837         }
 838
 839         /* Step 4: Concatenate the lengths of the AAD and source, and
 840          * hash that 16 byte buffer.
 841          */
 842         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 843                                    DMA_BIDIRECTIONAL);
 844         if (ret)
 845                 goto e_dst;
 846         final = (__be64 *)final_wa.address;
 847         final[0] = cpu_to_be64(aes->aad_len * 8);
 848         final[1] = cpu_to_be64(ilen * 8);
 849
 850         memset(&op, 0, sizeof(op));
 851         op.cmd_q = cmd_q;
 852         op.jobid = jobid;
 853         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 854         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 855         op.init = 1;
 856         op.u.aes.type = aes->type;
 857         op.u.aes.mode = CCP_AES_MODE_GHASH;
 858         op.u.aes.action = CCP_AES_GHASHFINAL;
 859         op.src.type = CCP_MEMTYPE_SYSTEM;
 860         op.src.u.dma.address = final_wa.dma.address;
 861         op.src.u.dma.length = AES_BLOCK_SIZE;
 862         op.dst.type = CCP_MEMTYPE_SYSTEM;
 863         op.dst.u.dma.address = final_wa.dma.address;
 864         op.dst.u.dma.length = AES_BLOCK_SIZE;
 865         op.eom = 1;
 866         op.u.aes.size = 0;
 867         ret = cmd_q->ccp->vdata->perform->aes(&op);
 868         if (ret)
 869                 goto e_final_wa;
 870
 871         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 872                 /* Put the ciphered tag after the ciphertext. */
 873                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 874         } else {
 875                 /* Does this ciphered tag match the input? */
 876                 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 877                                            DMA_BIDIRECTIONAL);
 878                 if (ret)
 879                         goto e_final_wa;
 880                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 881                 if (ret) {
 882                         ccp_dm_free(&tag);
 883                         goto e_final_wa;
 884                 }
 885
 886                 ret = crypto_memneq(tag.address, final_wa.address,
 887                                     authsize) ? -EBADMSG : 0;
 888                 ccp_dm_free(&tag);
 889         }
 890
 891 e_final_wa:
 892         ccp_dm_free(&final_wa);
 893
 894 e_dst:
 895         if (ilen > 0 && !in_place)
 896                 ccp_free_data(&dst, cmd_q);
 897
 898 e_src:
 899         if (ilen > 0)
 900                 ccp_free_data(&src, cmd_q);
 901
 902 e_aad:
 903         if (aes->aad_len)
 904                 ccp_free_data(&aad, cmd_q);
 905
 906 e_ctx:
 907         ccp_dm_free(&ctx);
 908
 909 e_key:
 910         ccp_dm_free(&key);
 911
 912         return ret;
 913 }
 914
 915 static noinline_for_stack int
 916 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 917 {
 918         struct ccp_aes_engine *aes = &cmd->u.aes;
 919         struct ccp_dm_workarea key, ctx;
 920         struct ccp_data src, dst;
 921         struct ccp_op op;
 922         unsigned int dm_offset;
 923         bool in_place = false;
 924         int ret;
 925
 926         if (!((aes->key_len == AES_KEYSIZE_128) ||
 927               (aes->key_len == AES_KEYSIZE_192) ||
 928               (aes->key_len == AES_KEYSIZE_256)))
 929                 return -EINVAL;
 930
 931         if (((aes->mode == CCP_AES_MODE_ECB) ||
 932              (aes->mode == CCP_AES_MODE_CBC)) &&
 933             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 934                 return -EINVAL;
 935
 936         if (!aes->key || !aes->src || !aes->dst)
 937                 return -EINVAL;
 938
 939         if (aes->mode != CCP_AES_MODE_ECB) {
 940                 if (aes->iv_len != AES_BLOCK_SIZE)
 941                         return -EINVAL;
 942
 943                 if (!aes->iv)
 944                         return -EINVAL;
 945         }
 946
 947         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 948         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 949
 950         ret = -EIO;
 951         memset(&op, 0, sizeof(op));
 952         op.cmd_q = cmd_q;
 953         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 954         op.sb_key = cmd_q->sb_key;
 955         op.sb_ctx = cmd_q->sb_ctx;
 956         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 957         op.u.aes.type = aes->type;
 958         op.u.aes.mode = aes->mode;
 959         op.u.aes.action = aes->action;
 960
 961         /* All supported key sizes fit in a single (32-byte) SB entry
 962          * and must be in little endian format. Use the 256-bit byte
 963          * swap passthru option to convert from big endian to little
 964          * endian.
 965          */
 966         ret = ccp_init_dm_workarea(&key, cmd_q,
 967                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 968                                    DMA_TO_DEVICE);
 969         if (ret)
 970                 return ret;
 971
 972         dm_offset = CCP_SB_BYTES - aes->key_len;
 973         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 974         if (ret)
 975                 goto e_key;
 976         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 977                              CCP_PASSTHRU_BYTESWAP_256BIT);
 978         if (ret) {
 979                 cmd->engine_error = cmd_q->cmd_error;
 980                 goto e_key;
 981         }
 982
 983         /* The AES context fits in a single (32-byte) SB entry and
 984          * must be in little endian format. Use the 256-bit byte swap
 985          * passthru option to convert from big endian to little endian.
 986          */
 987         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 988                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 989                                    DMA_BIDIRECTIONAL);
 990         if (ret)
 991                 goto e_key;
 992
 993         if (aes->mode != CCP_AES_MODE_ECB) {
 994                 /* Load the AES context - convert to LE */
 995                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 996                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 997                 if (ret)
 998                         goto e_ctx;
 999                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1000                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1001                 if (ret) {
1002                         cmd->engine_error = cmd_q->cmd_error;
1003                         goto e_ctx;
1004                 }
1005         }
1006         switch (aes->mode) {
1007         case CCP_AES_MODE_CFB: /* CFB128 only */
1008         case CCP_AES_MODE_CTR:
1009                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1010                 break;
1011         default:
1012                 op.u.aes.size = 0;
1013         }
1014
1015         /* Prepare the input and output data workareas. For in-place
1016          * operations we need to set the dma direction to BIDIRECTIONAL
1017          * and copy the src workarea to the dst workarea.
1018          */
1019         if (sg_virt(aes->src) == sg_virt(aes->dst))
1020                 in_place = true;
1021
1022         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1023                             AES_BLOCK_SIZE,
1024                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1025         if (ret)
1026                 goto e_ctx;
1027
1028         if (in_place) {
1029                 dst = src;
1030         } else {
1031                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1032                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1033                 if (ret)
1034                         goto e_src;
1035         }
1036
1037         /* Send data to the CCP AES engine */
1038         while (src.sg_wa.bytes_left) {
1039                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1040                 if (!src.sg_wa.bytes_left) {
1041                         op.eom = 1;
1042
1043                         /* Since we don't retrieve the AES context in ECB
1044                          * mode we have to wait for the operation to complete
1045                          * on the last piece of data
1046                          */
1047                         if (aes->mode == CCP_AES_MODE_ECB)
1048                                 op.soc = 1;
1049                 }
1050
1051                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1052                 if (ret) {
1053                         cmd->engine_error = cmd_q->cmd_error;
1054                         goto e_dst;
1055                 }
1056
1057                 ccp_process_data(&src, &dst, &op);
1058         }
1059
1060         if (aes->mode != CCP_AES_MODE_ECB) {
1061                 /* Retrieve the AES context - convert from LE to BE using
1062                  * 32-byte (256-bit) byteswapping
1063                  */
1064                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1065                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1066                 if (ret) {
1067                         cmd->engine_error = cmd_q->cmd_error;
1068                         goto e_dst;
1069                 }
1070
1071                 /* ...but we only need AES_BLOCK_SIZE bytes */
1072                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1073                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1074         }
1075
1076 e_dst:
1077         if (!in_place)
1078                 ccp_free_data(&dst, cmd_q);
1079
1080 e_src:
1081         ccp_free_data(&src, cmd_q);
1082
1083 e_ctx:
1084         ccp_dm_free(&ctx);
1085
1086 e_key:
1087         ccp_dm_free(&key);
1088
1089         return ret;
1090 }
1091
1092 static noinline_for_stack int
1093 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1094 {
1095         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1096         struct ccp_dm_workarea key, ctx;
1097         struct ccp_data src, dst;
1098         struct ccp_op op;
1099         unsigned int unit_size, dm_offset;
1100         bool in_place = false;
1101         unsigned int sb_count;
1102         enum ccp_aes_type aestype;
1103         int ret;
1104
1105         switch (xts->unit_size) {
1106         case CCP_XTS_AES_UNIT_SIZE_16:
1107                 unit_size = 16;
1108                 break;
1109         case CCP_XTS_AES_UNIT_SIZE_512:
1110                 unit_size = 512;
1111                 break;
1112         case CCP_XTS_AES_UNIT_SIZE_1024:
1113                 unit_size = 1024;
1114                 break;
1115         case CCP_XTS_AES_UNIT_SIZE_2048:
1116                 unit_size = 2048;
1117                 break;
1118         case CCP_XTS_AES_UNIT_SIZE_4096:
1119                 unit_size = 4096;
1120                 break;
1121
1122         default:
1123                 return -EINVAL;
1124         }
1125
1126         if (xts->key_len == AES_KEYSIZE_128)
1127                 aestype = CCP_AES_TYPE_128;
1128         else if (xts->key_len == AES_KEYSIZE_256)
1129                 aestype = CCP_AES_TYPE_256;
1130         else
1131                 return -EINVAL;
1132
1133         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1134                 return -EINVAL;
1135
1136         if (xts->iv_len != AES_BLOCK_SIZE)
1137                 return -EINVAL;
1138
1139         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1140                 return -EINVAL;
1141
1142         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1143         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1144
1145         ret = -EIO;
1146         memset(&op, 0, sizeof(op));
1147         op.cmd_q = cmd_q;
1148         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1149         op.sb_key = cmd_q->sb_key;
1150         op.sb_ctx = cmd_q->sb_ctx;
1151         op.init = 1;
1152         op.u.xts.type = aestype;
1153         op.u.xts.action = xts->action;
1154         op.u.xts.unit_size = xts->unit_size;
1155
1156         /* A version 3 device only supports 128-bit keys, which fits into a
1157          * single SB entry. A version 5 device uses a 512-bit vector, so two
1158          * SB entries.
1159          */
1160         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1161                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1162         else
1163                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1164         ret = ccp_init_dm_workarea(&key, cmd_q,
1165                                    sb_count * CCP_SB_BYTES,
1166                                    DMA_TO_DEVICE);
1167         if (ret)
1168                 return ret;
1169
1170         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1171                 /* All supported key sizes must be in little endian format.
1172                  * Use the 256-bit byte swap passthru option to convert from
1173                  * big endian to little endian.
1174                  */
1175                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1176                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1177                 if (ret)
1178                         goto e_key;
1179                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1180                 if (ret)
1181                         goto e_key;
1182         } else {
1183                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1184                  * occupies 256 bits, or one entire slot, and is zero-padded.
1185                  */
1186                 unsigned int pad;
1187
1188                 dm_offset = CCP_SB_BYTES;
1189                 pad = dm_offset - xts->key_len;
1190                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1191                 if (ret)
1192                         goto e_key;
1193                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1194                                       xts->key_len, xts->key_len);
1195                 if (ret)
1196                         goto e_key;
1197         }
1198         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1199                              CCP_PASSTHRU_BYTESWAP_256BIT);
1200         if (ret) {
1201                 cmd->engine_error = cmd_q->cmd_error;
1202                 goto e_key;
1203         }
1204
1205         /* The AES context fits in a single (32-byte) SB entry and
1206          * for XTS is already in little endian format so no byte swapping
1207          * is needed.
1208          */
1209         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1210                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1211                                    DMA_BIDIRECTIONAL);
1212         if (ret)
1213                 goto e_key;
1214
1215         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1216         if (ret)
1217                 goto e_ctx;
1218         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1219                              CCP_PASSTHRU_BYTESWAP_NOOP);
1220         if (ret) {
1221                 cmd->engine_error = cmd_q->cmd_error;
1222                 goto e_ctx;
1223         }
1224
1225         /* Prepare the input and output data workareas. For in-place
1226          * operations we need to set the dma direction to BIDIRECTIONAL
1227          * and copy the src workarea to the dst workarea.
1228          */
1229         if (sg_virt(xts->src) == sg_virt(xts->dst))
1230                 in_place = true;
1231
1232         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1233                             unit_size,
1234                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1235         if (ret)
1236                 goto e_ctx;
1237
1238         if (in_place) {
1239                 dst = src;
1240         } else {
1241                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1242                                     unit_size, DMA_FROM_DEVICE);
1243                 if (ret)
1244                         goto e_src;
1245         }
1246
1247         /* Send data to the CCP AES engine */
1248         while (src.sg_wa.bytes_left) {
1249                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1250                 if (!src.sg_wa.bytes_left)
1251                         op.eom = 1;
1252
1253                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1254                 if (ret) {
1255                         cmd->engine_error = cmd_q->cmd_error;
1256                         goto e_dst;
1257                 }
1258
1259                 ccp_process_data(&src, &dst, &op);
1260         }
1261
1262         /* Retrieve the AES context - convert from LE to BE using
1263          * 32-byte (256-bit) byteswapping
1264          */
1265         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1266                                CCP_PASSTHRU_BYTESWAP_256BIT);
1267         if (ret) {
1268                 cmd->engine_error = cmd_q->cmd_error;
1269                 goto e_dst;
1270         }
1271
1272         /* ...but we only need AES_BLOCK_SIZE bytes */
1273         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1274         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1275
1276 e_dst:
1277         if (!in_place)
1278                 ccp_free_data(&dst, cmd_q);
1279
1280 e_src:
1281         ccp_free_data(&src, cmd_q);
1282
1283 e_ctx:
1284         ccp_dm_free(&ctx);
1285
1286 e_key:
1287         ccp_dm_free(&key);
1288
1289         return ret;
1290 }
1291
1292 static noinline_for_stack int
1293 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1294 {
1295         struct ccp_des3_engine *des3 = &cmd->u.des3;
1296
1297         struct ccp_dm_workarea key, ctx;
1298         struct ccp_data src, dst;
1299         struct ccp_op op;
1300         unsigned int dm_offset;
1301         unsigned int len_singlekey;
1302         bool in_place = false;
1303         int ret;
1304
1305         /* Error checks */
1306         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1307                 return -EINVAL;
1308
1309         if (!cmd_q->ccp->vdata->perform->des3)
1310                 return -EINVAL;
1311
1312         if (des3->key_len != DES3_EDE_KEY_SIZE)
1313                 return -EINVAL;
1314
1315         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1316                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1317                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1318                 return -EINVAL;
1319
1320         if (!des3->key || !des3->src || !des3->dst)
1321                 return -EINVAL;
1322
1323         if (des3->mode != CCP_DES3_MODE_ECB) {
1324                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1325                         return -EINVAL;
1326
1327                 if (!des3->iv)
1328                         return -EINVAL;
1329         }
1330
1331         /* Zero out all the fields of the command desc */
1332         memset(&op, 0, sizeof(op));
1333
1334         /* Set up the Function field */
1335         op.cmd_q = cmd_q;
1336         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1337         op.sb_key = cmd_q->sb_key;
1338
1339         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1340         op.u.des3.type = des3->type;
1341         op.u.des3.mode = des3->mode;
1342         op.u.des3.action = des3->action;
1343
1344         /*
1345          * All supported key sizes fit in a single (32-byte) KSB entry and
1346          * (like AES) must be in little endian format. Use the 256-bit byte
1347          * swap passthru option to convert from big endian to little endian.
1348          */
1349         ret = ccp_init_dm_workarea(&key, cmd_q,
1350                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1351                                    DMA_TO_DEVICE);
1352         if (ret)
1353                 return ret;
1354
1355         /*
1356          * The contents of the key triplet are in the reverse order of what
1357          * is required by the engine. Copy the 3 pieces individually to put
1358          * them where they belong.
1359          */
1360         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1361
1362         len_singlekey = des3->key_len / 3;
1363         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1364                               des3->key, 0, len_singlekey);
1365         if (ret)
1366                 goto e_key;
1367         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1368                               des3->key, len_singlekey, len_singlekey);
1369         if (ret)
1370                 goto e_key;
1371         ret = ccp_set_dm_area(&key, dm_offset,
1372                               des3->key, 2 * len_singlekey, len_singlekey);
1373         if (ret)
1374                 goto e_key;
1375
1376         /* Copy the key to the SB */
1377         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1378                              CCP_PASSTHRU_BYTESWAP_256BIT);
1379         if (ret) {
1380                 cmd->engine_error = cmd_q->cmd_error;
1381                 goto e_key;
1382         }
1383
1384         /*
1385          * The DES3 context fits in a single (32-byte) KSB entry and
1386          * must be in little endian format. Use the 256-bit byte swap
1387          * passthru option to convert from big endian to little endian.
1388          */
1389         if (des3->mode != CCP_DES3_MODE_ECB) {
1390                 op.sb_ctx = cmd_q->sb_ctx;
1391
1392                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1393                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1394                                            DMA_BIDIRECTIONAL);
1395                 if (ret)
1396                         goto e_key;
1397
1398                 /* Load the context into the LSB */
1399                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1400                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1401                                       des3->iv_len);
1402                 if (ret)
1403                         goto e_ctx;
1404
1405                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1406                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1407                 if (ret) {
1408                         cmd->engine_error = cmd_q->cmd_error;
1409                         goto e_ctx;
1410                 }
1411         }
1412
1413         /*
1414          * Prepare the input and output data workareas. For in-place
1415          * operations we need to set the dma direction to BIDIRECTIONAL
1416          * and copy the src workarea to the dst workarea.
1417          */
1418         if (sg_virt(des3->src) == sg_virt(des3->dst))
1419                 in_place = true;
1420
1421         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1422                         DES3_EDE_BLOCK_SIZE,
1423                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1424         if (ret)
1425                 goto e_ctx;
1426
1427         if (in_place)
1428                 dst = src;
1429         else {
1430                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1431                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1432                 if (ret)
1433                         goto e_src;
1434         }
1435
1436         /* Send data to the CCP DES3 engine */
1437         while (src.sg_wa.bytes_left) {
1438                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1439                 if (!src.sg_wa.bytes_left) {
1440                         op.eom = 1;
1441
1442                         /* Since we don't retrieve the context in ECB mode
1443                          * we have to wait for the operation to complete
1444                          * on the last piece of data
1445                          */
1446                         op.soc = 0;
1447                 }
1448
1449                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1450                 if (ret) {
1451                         cmd->engine_error = cmd_q->cmd_error;
1452                         goto e_dst;
1453                 }
1454
1455                 ccp_process_data(&src, &dst, &op);
1456         }
1457
1458         if (des3->mode != CCP_DES3_MODE_ECB) {
1459                 /* Retrieve the context and make BE */
1460                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1461                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1462                 if (ret) {
1463                         cmd->engine_error = cmd_q->cmd_error;
1464                         goto e_dst;
1465                 }
1466
1467                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1468                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1469                                 DES3_EDE_BLOCK_SIZE);
1470         }
1471 e_dst:
1472         if (!in_place)
1473                 ccp_free_data(&dst, cmd_q);
1474
1475 e_src:
1476         ccp_free_data(&src, cmd_q);
1477
1478 e_ctx:
1479         if (des3->mode != CCP_DES3_MODE_ECB)
1480                 ccp_dm_free(&ctx);
1481
1482 e_key:
1483         ccp_dm_free(&key);
1484
1485         return ret;
1486 }
1487
1488 static noinline_for_stack int
1489 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1490 {
1491         struct ccp_sha_engine *sha = &cmd->u.sha;
1492         struct ccp_dm_workarea ctx;
1493         struct ccp_data src;
1494         struct ccp_op op;
1495         unsigned int ioffset, ooffset;
1496         unsigned int digest_size;
1497         int sb_count;
1498         const void *init;
1499         u64 block_size;
1500         int ctx_size;
1501         int ret;
1502
1503         switch (sha->type) {
1504         case CCP_SHA_TYPE_1:
1505                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1506                         return -EINVAL;
1507                 block_size = SHA1_BLOCK_SIZE;
1508                 break;
1509         case CCP_SHA_TYPE_224:
1510                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1511                         return -EINVAL;
1512                 block_size = SHA224_BLOCK_SIZE;
1513                 break;
1514         case CCP_SHA_TYPE_256:
1515                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1516                         return -EINVAL;
1517                 block_size = SHA256_BLOCK_SIZE;
1518                 break;
1519         case CCP_SHA_TYPE_384:
1520                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1521                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1522                         return -EINVAL;
1523                 block_size = SHA384_BLOCK_SIZE;
1524                 break;
1525         case CCP_SHA_TYPE_512:
1526                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1527                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1528                         return -EINVAL;
1529                 block_size = SHA512_BLOCK_SIZE;
1530                 break;
1531         default:
1532                 return -EINVAL;
1533         }
1534
1535         if (!sha->ctx)
1536                 return -EINVAL;
1537
1538         if (!sha->final && (sha->src_len & (block_size - 1)))
1539                 return -EINVAL;
1540
1541         /* The version 3 device can't handle zero-length input */
1542         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1543
1544                 if (!sha->src_len) {
1545                         unsigned int digest_len;
1546                         const u8 *sha_zero;
1547
1548                         /* Not final, just return */
1549                         if (!sha->final)
1550                                 return 0;
1551
1552                         /* CCP can't do a zero length sha operation so the
1553                          * caller must buffer the data.
1554                          */
1555                         if (sha->msg_bits)
1556                                 return -EINVAL;
1557
1558                         /* The CCP cannot perform zero-length sha operations
1559                          * so the caller is required to buffer data for the
1560                          * final operation. However, a sha operation for a
1561                          * message with a total length of zero is valid so
1562                          * known values are required to supply the result.
1563                          */
1564                         switch (sha->type) {
1565                         case CCP_SHA_TYPE_1:
1566                                 sha_zero = sha1_zero_message_hash;
1567                                 digest_len = SHA1_DIGEST_SIZE;
1568                                 break;
1569                         case CCP_SHA_TYPE_224:
1570                                 sha_zero = sha224_zero_message_hash;
1571                                 digest_len = SHA224_DIGEST_SIZE;
1572                                 break;
1573                         case CCP_SHA_TYPE_256:
1574                                 sha_zero = sha256_zero_message_hash;
1575                                 digest_len = SHA256_DIGEST_SIZE;
1576                                 break;
1577                         default:
1578                                 return -EINVAL;
1579                         }
1580
1581                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1582                                                  digest_len, 1);
1583
1584                         return 0;
1585                 }
1586         }
1587
1588         /* Set variables used throughout */
1589         switch (sha->type) {
1590         case CCP_SHA_TYPE_1:
1591                 digest_size = SHA1_DIGEST_SIZE;
1592                 init = (void *) ccp_sha1_init;
1593                 ctx_size = SHA1_DIGEST_SIZE;
1594                 sb_count = 1;
1595                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1596                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1597                 else
1598                         ooffset = ioffset = 0;
1599                 break;
1600         case CCP_SHA_TYPE_224:
1601                 digest_size = SHA224_DIGEST_SIZE;
1602                 init = (void *) ccp_sha224_init;
1603                 ctx_size = SHA256_DIGEST_SIZE;
1604                 sb_count = 1;
1605                 ioffset = 0;
1606                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1607                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1608                 else
1609                         ooffset = 0;
1610                 break;
1611         case CCP_SHA_TYPE_256:
1612                 digest_size = SHA256_DIGEST_SIZE;
1613                 init = (void *) ccp_sha256_init;
1614                 ctx_size = SHA256_DIGEST_SIZE;
1615                 sb_count = 1;
1616                 ooffset = ioffset = 0;
1617                 break;
1618         case CCP_SHA_TYPE_384:
1619                 digest_size = SHA384_DIGEST_SIZE;
1620                 init = (void *) ccp_sha384_init;
1621                 ctx_size = SHA512_DIGEST_SIZE;
1622                 sb_count = 2;
1623                 ioffset = 0;
1624                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1625                 break;
1626         case CCP_SHA_TYPE_512:
1627                 digest_size = SHA512_DIGEST_SIZE;
1628                 init = (void *) ccp_sha512_init;
1629                 ctx_size = SHA512_DIGEST_SIZE;
1630                 sb_count = 2;
1631                 ooffset = ioffset = 0;
1632                 break;
1633         default:
1634                 ret = -EINVAL;
1635                 goto e_data;
1636         }
1637
1638         /* For zero-length plaintext the src pointer is ignored;
1639          * otherwise both parts must be valid
1640          */
1641         if (sha->src_len && !sha->src)
1642                 return -EINVAL;
1643
1644         memset(&op, 0, sizeof(op));
1645         op.cmd_q = cmd_q;
1646         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1647         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1648         op.u.sha.type = sha->type;
1649         op.u.sha.msg_bits = sha->msg_bits;
1650
1651         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1652          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1653          * first slot, and the left half in the second. Each portion must then
1654          * be in little endian format: use the 256-bit byte swap option.
1655          */
1656         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1657                                    DMA_BIDIRECTIONAL);
1658         if (ret)
1659                 return ret;
1660         if (sha->first) {
1661                 switch (sha->type) {
1662                 case CCP_SHA_TYPE_1:
1663                 case CCP_SHA_TYPE_224:
1664                 case CCP_SHA_TYPE_256:
1665                         memcpy(ctx.address + ioffset, init, ctx_size);
1666                         break;
1667                 case CCP_SHA_TYPE_384:
1668                 case CCP_SHA_TYPE_512:
1669                         memcpy(ctx.address + ctx_size / 2, init,
1670                                ctx_size / 2);
1671                         memcpy(ctx.address, init + ctx_size / 2,
1672                                ctx_size / 2);
1673                         break;
1674                 default:
1675                         ret = -EINVAL;
1676                         goto e_ctx;
1677                 }
1678         } else {
1679                 /* Restore the context */
1680                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1681                                       sb_count * CCP_SB_BYTES);
1682                 if (ret)
1683                         goto e_ctx;
1684         }
1685
1686         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1687                              CCP_PASSTHRU_BYTESWAP_256BIT);
1688         if (ret) {
1689                 cmd->engine_error = cmd_q->cmd_error;
1690                 goto e_ctx;
1691         }
1692
1693         if (sha->src) {
1694                 /* Send data to the CCP SHA engine; block_size is set above */
1695                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1696                                     block_size, DMA_TO_DEVICE);
1697                 if (ret)
1698                         goto e_ctx;
1699
1700                 while (src.sg_wa.bytes_left) {
1701                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1702                         if (sha->final && !src.sg_wa.bytes_left)
1703                                 op.eom = 1;
1704
1705                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1706                         if (ret) {
1707                                 cmd->engine_error = cmd_q->cmd_error;
1708                                 goto e_data;
1709                         }
1710
1711                         ccp_process_data(&src, NULL, &op);
1712                 }
1713         } else {
1714                 op.eom = 1;
1715                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1716                 if (ret) {
1717                         cmd->engine_error = cmd_q->cmd_error;
1718                         goto e_data;
1719                 }
1720         }
1721
1722         /* Retrieve the SHA context - convert from LE to BE using
1723          * 32-byte (256-bit) byteswapping to BE
1724          */
1725         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1726                                CCP_PASSTHRU_BYTESWAP_256BIT);
1727         if (ret) {
1728                 cmd->engine_error = cmd_q->cmd_error;
1729                 goto e_data;
1730         }
1731
1732         if (sha->final) {
1733                 /* Finishing up, so get the digest */
1734                 switch (sha->type) {
1735                 case CCP_SHA_TYPE_1:
1736                 case CCP_SHA_TYPE_224:
1737                 case CCP_SHA_TYPE_256:
1738                         ccp_get_dm_area(&ctx, ooffset,
1739                                         sha->ctx, 0,
1740                                         digest_size);
1741                         break;
1742                 case CCP_SHA_TYPE_384:
1743                 case CCP_SHA_TYPE_512:
1744                         ccp_get_dm_area(&ctx, 0,
1745                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1746                                         LSB_ITEM_SIZE);
1747                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1748                                         sha->ctx, 0,
1749                                         LSB_ITEM_SIZE - ooffset);
1750                         break;
1751                 default:
1752                         ret = -EINVAL;
1753                         goto e_data;
1754                 }
1755         } else {
1756                 /* Stash the context */
1757                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1758                                 sb_count * CCP_SB_BYTES);
1759         }
1760
1761         if (sha->final && sha->opad) {
1762                 /* HMAC operation, recursively perform final SHA */
1763                 struct ccp_cmd hmac_cmd;
1764                 struct scatterlist sg;
1765                 u8 *hmac_buf;
1766
1767                 if (sha->opad_len != block_size) {
1768                         ret = -EINVAL;
1769                         goto e_data;
1770                 }
1771
1772                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1773                 if (!hmac_buf) {
1774                         ret = -ENOMEM;
1775                         goto e_data;
1776                 }
1777                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1778
1779                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1780                 switch (sha->type) {
1781                 case CCP_SHA_TYPE_1:
1782                 case CCP_SHA_TYPE_224:
1783                 case CCP_SHA_TYPE_256:
1784                         memcpy(hmac_buf + block_size,
1785                                ctx.address + ooffset,
1786                                digest_size);
1787                         break;
1788                 case CCP_SHA_TYPE_384:
1789                 case CCP_SHA_TYPE_512:
1790                         memcpy(hmac_buf + block_size,
1791                                ctx.address + LSB_ITEM_SIZE + ooffset,
1792                                LSB_ITEM_SIZE);
1793                         memcpy(hmac_buf + block_size +
1794                                (LSB_ITEM_SIZE - ooffset),
1795                                ctx.address,
1796                                LSB_ITEM_SIZE);
1797                         break;
1798                 default:
1799                         kfree(hmac_buf);
1800                         ret = -EINVAL;
1801                         goto e_data;
1802                 }
1803
1804                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1805                 hmac_cmd.engine = CCP_ENGINE_SHA;
1806                 hmac_cmd.u.sha.type = sha->type;
1807                 hmac_cmd.u.sha.ctx = sha->ctx;
1808                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1809                 hmac_cmd.u.sha.src = &sg;
1810                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1811                 hmac_cmd.u.sha.opad = NULL;
1812                 hmac_cmd.u.sha.opad_len = 0;
1813                 hmac_cmd.u.sha.first = 1;
1814                 hmac_cmd.u.sha.final = 1;
1815                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1816
1817                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1818                 if (ret)
1819                         cmd->engine_error = hmac_cmd.engine_error;
1820
1821                 kfree(hmac_buf);
1822         }
1823
1824 e_data:
1825         if (sha->src)
1826                 ccp_free_data(&src, cmd_q);
1827
1828 e_ctx:
1829         ccp_dm_free(&ctx);
1830
1831         return ret;
1832 }
1833
1834 static noinline_for_stack int
1835 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1836 {
1837         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1838         struct ccp_dm_workarea exp, src, dst;
1839         struct ccp_op op;
1840         unsigned int sb_count, i_len, o_len;
1841         int ret;
1842
1843         /* Check against the maximum allowable size, in bits */
1844         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1845                 return -EINVAL;
1846
1847         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1848                 return -EINVAL;
1849
1850         memset(&op, 0, sizeof(op));
1851         op.cmd_q = cmd_q;
1852         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1853
1854         /* The RSA modulus must precede the message being acted upon, so
1855          * it must be copied to a DMA area where the message and the
1856          * modulus can be concatenated.  Therefore the input buffer
1857          * length required is twice the output buffer length (which
1858          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1859          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1860          * required.
1861          */
1862         o_len = 32 * ((rsa->key_size + 255) / 256);
1863         i_len = o_len * 2;
1864
1865         sb_count = 0;
1866         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1867                 /* sb_count is the number of storage block slots required
1868                  * for the modulus.
1869                  */
1870                 sb_count = o_len / CCP_SB_BYTES;
1871                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1872                                                                 sb_count);
1873                 if (!op.sb_key)
1874                         return -EIO;
1875         } else {
1876                 /* A version 5 device allows a modulus size that will not fit
1877                  * in the LSB, so the command will transfer it from memory.
1878                  * Set the sb key to the default, even though it's not used.
1879                  */
1880                 op.sb_key = cmd_q->sb_key;
1881         }
1882
1883         /* The RSA exponent must be in little endian format. Reverse its
1884          * byte order.
1885          */
1886         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1887         if (ret)
1888                 goto e_sb;
1889
1890         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1891         if (ret)
1892                 goto e_exp;
1893
1894         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1895                 /* Copy the exponent to the local storage block, using
1896                  * as many 32-byte blocks as were allocated above. It's
1897                  * already little endian, so no further change is required.
1898                  */
1899                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1900                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1901                 if (ret) {
1902                         cmd->engine_error = cmd_q->cmd_error;
1903                         goto e_exp;
1904                 }
1905         } else {
1906                 /* The exponent can be retrieved from memory via DMA. */
1907                 op.exp.u.dma.address = exp.dma.address;
1908                 op.exp.u.dma.offset = 0;
1909         }
1910
1911         /* Concatenate the modulus and the message. Both the modulus and
1912          * the operands must be in little endian format.  Since the input
1913          * is in big endian format it must be converted.
1914          */
1915         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1916         if (ret)
1917                 goto e_exp;
1918
1919         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1920         if (ret)
1921                 goto e_src;
1922         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1923         if (ret)
1924                 goto e_src;
1925
1926         /* Prepare the output area for the operation */
1927         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1928         if (ret)
1929                 goto e_src;
1930
1931         op.soc = 1;
1932         op.src.u.dma.address = src.dma.address;
1933         op.src.u.dma.offset = 0;
1934         op.src.u.dma.length = i_len;
1935         op.dst.u.dma.address = dst.dma.address;
1936         op.dst.u.dma.offset = 0;
1937         op.dst.u.dma.length = o_len;
1938
1939         op.u.rsa.mod_size = rsa->key_size;
1940         op.u.rsa.input_len = i_len;
1941
1942         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1943         if (ret) {
1944                 cmd->engine_error = cmd_q->cmd_error;
1945                 goto e_dst;
1946         }
1947
1948         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1949
1950 e_dst:
1951         ccp_dm_free(&dst);
1952
1953 e_src:
1954         ccp_dm_free(&src);
1955
1956 e_exp:
1957         ccp_dm_free(&exp);
1958
1959 e_sb:
1960         if (sb_count)
1961                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1962
1963         return ret;
1964 }
1965
1966 static noinline_for_stack int
1967 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1968 {
1969         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1970         struct ccp_dm_workarea mask;
1971         struct ccp_data src, dst;
1972         struct ccp_op op;
1973         bool in_place = false;
1974         unsigned int i;
1975         int ret = 0;
1976
1977         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1978                 return -EINVAL;
1979
1980         if (!pt->src || !pt->dst)
1981                 return -EINVAL;
1982
1983         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1984                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1985                         return -EINVAL;
1986                 if (!pt->mask)
1987                         return -EINVAL;
1988         }
1989
1990         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1991
1992         memset(&op, 0, sizeof(op));
1993         op.cmd_q = cmd_q;
1994         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1995
1996         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1997                 /* Load the mask */
1998                 op.sb_key = cmd_q->sb_key;
1999
2000                 ret = ccp_init_dm_workarea(&mask, cmd_q,
2001                                            CCP_PASSTHRU_SB_COUNT *
2002                                            CCP_SB_BYTES,
2003                                            DMA_TO_DEVICE);
2004                 if (ret)
2005                         return ret;
2006
2007                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2008                 if (ret)
2009                         goto e_mask;
2010                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2011                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2012                 if (ret) {
2013                         cmd->engine_error = cmd_q->cmd_error;
2014                         goto e_mask;
2015                 }
2016         }
2017
2018         /* Prepare the input and output data workareas. For in-place
2019          * operations we need to set the dma direction to BIDIRECTIONAL
2020          * and copy the src workarea to the dst workarea.
2021          */
2022         if (sg_virt(pt->src) == sg_virt(pt->dst))
2023                 in_place = true;
2024
2025         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2026                             CCP_PASSTHRU_MASKSIZE,
2027                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2028         if (ret)
2029                 goto e_mask;
2030
2031         if (in_place) {
2032                 dst = src;
2033         } else {
2034                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2035                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2036                 if (ret)
2037                         goto e_src;
2038         }
2039
2040         /* Send data to the CCP Passthru engine
2041          *   Because the CCP engine works on a single source and destination
2042          *   dma address at a time, each entry in the source scatterlist
2043          *   (after the dma_map_sg call) must be less than or equal to the
2044          *   (remaining) length in the destination scatterlist entry and the
2045          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2046          */
2047         dst.sg_wa.sg_used = 0;
2048         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2049                 if (!dst.sg_wa.sg ||
2050                     (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2051                         ret = -EINVAL;
2052                         goto e_dst;
2053                 }
2054
2055                 if (i == src.sg_wa.dma_count) {
2056                         op.eom = 1;
2057                         op.soc = 1;
2058                 }
2059
2060                 op.src.type = CCP_MEMTYPE_SYSTEM;
2061                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2062                 op.src.u.dma.offset = 0;
2063                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2064
2065                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2066                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2067                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2068                 op.dst.u.dma.length = op.src.u.dma.length;
2069
2070                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2071                 if (ret) {
2072                         cmd->engine_error = cmd_q->cmd_error;
2073                         goto e_dst;
2074                 }
2075
2076                 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2077                 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2078                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2079                         dst.sg_wa.sg_used = 0;
2080                 }
2081                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2082         }
2083
2084 e_dst:
2085         if (!in_place)
2086                 ccp_free_data(&dst, cmd_q);
2087
2088 e_src:
2089         ccp_free_data(&src, cmd_q);
2090
2091 e_mask:
2092         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2093                 ccp_dm_free(&mask);
2094
2095         return ret;
2096 }
2097
2098 static noinline_for_stack int
2099 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2100                                       struct ccp_cmd *cmd)
2101 {
2102         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2103         struct ccp_dm_workarea mask;
2104         struct ccp_op op;
2105         int ret;
2106
2107         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2108                 return -EINVAL;
2109
2110         if (!pt->src_dma || !pt->dst_dma)
2111                 return -EINVAL;
2112
2113         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2114                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2115                         return -EINVAL;
2116                 if (!pt->mask)
2117                         return -EINVAL;
2118         }
2119
2120         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2121
2122         memset(&op, 0, sizeof(op));
2123         op.cmd_q = cmd_q;
2124         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2125
2126         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2127                 /* Load the mask */
2128                 op.sb_key = cmd_q->sb_key;
2129
2130                 mask.length = pt->mask_len;
2131                 mask.dma.address = pt->mask;
2132                 mask.dma.length = pt->mask_len;
2133
2134                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2135                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2136                 if (ret) {
2137                         cmd->engine_error = cmd_q->cmd_error;
2138                         return ret;
2139                 }
2140         }
2141
2142         /* Send data to the CCP Passthru engine */
2143         op.eom = 1;
2144         op.soc = 1;
2145
2146         op.src.type = CCP_MEMTYPE_SYSTEM;
2147         op.src.u.dma.address = pt->src_dma;
2148         op.src.u.dma.offset = 0;
2149         op.src.u.dma.length = pt->src_len;
2150
2151         op.dst.type = CCP_MEMTYPE_SYSTEM;
2152         op.dst.u.dma.address = pt->dst_dma;
2153         op.dst.u.dma.offset = 0;
2154         op.dst.u.dma.length = pt->src_len;
2155
2156         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2157         if (ret)
2158                 cmd->engine_error = cmd_q->cmd_error;
2159
2160         return ret;
2161 }
2162
2163 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2164 {
2165         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2166         struct ccp_dm_workarea src, dst;
2167         struct ccp_op op;
2168         int ret;
2169         u8 *save;
2170
2171         if (!ecc->u.mm.operand_1 ||
2172             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2173                 return -EINVAL;
2174
2175         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2176                 if (!ecc->u.mm.operand_2 ||
2177                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2178                         return -EINVAL;
2179
2180         if (!ecc->u.mm.result ||
2181             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2182                 return -EINVAL;
2183
2184         memset(&op, 0, sizeof(op));
2185         op.cmd_q = cmd_q;
2186         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2187
2188         /* Concatenate the modulus and the operands. Both the modulus and
2189          * the operands must be in little endian format.  Since the input
2190          * is in big endian format it must be converted and placed in a
2191          * fixed length buffer.
2192          */
2193         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2194                                    DMA_TO_DEVICE);
2195         if (ret)
2196                 return ret;
2197
2198         /* Save the workarea address since it is updated in order to perform
2199          * the concatenation
2200          */
2201         save = src.address;
2202
2203         /* Copy the ECC modulus */
2204         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2205         if (ret)
2206                 goto e_src;
2207         src.address += CCP_ECC_OPERAND_SIZE;
2208
2209         /* Copy the first operand */
2210         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2211                                       ecc->u.mm.operand_1_len);
2212         if (ret)
2213                 goto e_src;
2214         src.address += CCP_ECC_OPERAND_SIZE;
2215
2216         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2217                 /* Copy the second operand */
2218                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2219                                               ecc->u.mm.operand_2_len);
2220                 if (ret)
2221                         goto e_src;
2222                 src.address += CCP_ECC_OPERAND_SIZE;
2223         }
2224
2225         /* Restore the workarea address */
2226         src.address = save;
2227
2228         /* Prepare the output area for the operation */
2229         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2230                                    DMA_FROM_DEVICE);
2231         if (ret)
2232                 goto e_src;
2233
2234         op.soc = 1;
2235         op.src.u.dma.address = src.dma.address;
2236         op.src.u.dma.offset = 0;
2237         op.src.u.dma.length = src.length;
2238         op.dst.u.dma.address = dst.dma.address;
2239         op.dst.u.dma.offset = 0;
2240         op.dst.u.dma.length = dst.length;
2241
2242         op.u.ecc.function = cmd->u.ecc.function;
2243
2244         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2245         if (ret) {
2246                 cmd->engine_error = cmd_q->cmd_error;
2247                 goto e_dst;
2248         }
2249
2250         ecc->ecc_result = le16_to_cpup(
2251                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2252         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2253                 ret = -EIO;
2254                 goto e_dst;
2255         }
2256
2257         /* Save the ECC result */
2258         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2259                                 CCP_ECC_MODULUS_BYTES);
2260
2261 e_dst:
2262         ccp_dm_free(&dst);
2263
2264 e_src:
2265         ccp_dm_free(&src);
2266
2267         return ret;
2268 }
2269
2270 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2271 {
2272         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2273         struct ccp_dm_workarea src, dst;
2274         struct ccp_op op;
2275         int ret;
2276         u8 *save;
2277
2278         if (!ecc->u.pm.point_1.x ||
2279             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2280             !ecc->u.pm.point_1.y ||
2281             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2282                 return -EINVAL;
2283
2284         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2285                 if (!ecc->u.pm.point_2.x ||
2286                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2287                     !ecc->u.pm.point_2.y ||
2288                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2289                         return -EINVAL;
2290         } else {
2291                 if (!ecc->u.pm.domain_a ||
2292                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2293                         return -EINVAL;
2294
2295                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2296                         if (!ecc->u.pm.scalar ||
2297                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2298                                 return -EINVAL;
2299         }
2300
2301         if (!ecc->u.pm.result.x ||
2302             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2303             !ecc->u.pm.result.y ||
2304             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2305                 return -EINVAL;
2306
2307         memset(&op, 0, sizeof(op));
2308         op.cmd_q = cmd_q;
2309         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2310
2311         /* Concatenate the modulus and the operands. Both the modulus and
2312          * the operands must be in little endian format.  Since the input
2313          * is in big endian format it must be converted and placed in a
2314          * fixed length buffer.
2315          */
2316         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2317                                    DMA_TO_DEVICE);
2318         if (ret)
2319                 return ret;
2320
2321         /* Save the workarea address since it is updated in order to perform
2322          * the concatenation
2323          */
2324         save = src.address;
2325
2326         /* Copy the ECC modulus */
2327         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2328         if (ret)
2329                 goto e_src;
2330         src.address += CCP_ECC_OPERAND_SIZE;
2331
2332         /* Copy the first point X and Y coordinate */
2333         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2334                                       ecc->u.pm.point_1.x_len);
2335         if (ret)
2336                 goto e_src;
2337         src.address += CCP_ECC_OPERAND_SIZE;
2338         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2339                                       ecc->u.pm.point_1.y_len);
2340         if (ret)
2341                 goto e_src;
2342         src.address += CCP_ECC_OPERAND_SIZE;
2343
2344         /* Set the first point Z coordinate to 1 */
2345         *src.address = 0x01;
2346         src.address += CCP_ECC_OPERAND_SIZE;
2347
2348         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2349                 /* Copy the second point X and Y coordinate */
2350                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2351                                               ecc->u.pm.point_2.x_len);
2352                 if (ret)
2353                         goto e_src;
2354                 src.address += CCP_ECC_OPERAND_SIZE;
2355                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2356                                               ecc->u.pm.point_2.y_len);
2357                 if (ret)
2358                         goto e_src;
2359                 src.address += CCP_ECC_OPERAND_SIZE;
2360
2361                 /* Set the second point Z coordinate to 1 */
2362                 *src.address = 0x01;
2363                 src.address += CCP_ECC_OPERAND_SIZE;
2364         } else {
2365                 /* Copy the Domain "a" parameter */
2366                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2367                                               ecc->u.pm.domain_a_len);
2368                 if (ret)
2369                         goto e_src;
2370                 src.address += CCP_ECC_OPERAND_SIZE;
2371
2372                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2373                         /* Copy the scalar value */
2374                         ret = ccp_reverse_set_dm_area(&src, 0,
2375                                                       ecc->u.pm.scalar, 0,
2376                                                       ecc->u.pm.scalar_len);
2377                         if (ret)
2378                                 goto e_src;
2379                         src.address += CCP_ECC_OPERAND_SIZE;
2380                 }
2381         }
2382
2383         /* Restore the workarea address */
2384         src.address = save;
2385
2386         /* Prepare the output area for the operation */
2387         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2388                                    DMA_FROM_DEVICE);
2389         if (ret)
2390                 goto e_src;
2391
2392         op.soc = 1;
2393         op.src.u.dma.address = src.dma.address;
2394         op.src.u.dma.offset = 0;
2395         op.src.u.dma.length = src.length;
2396         op.dst.u.dma.address = dst.dma.address;
2397         op.dst.u.dma.offset = 0;
2398         op.dst.u.dma.length = dst.length;
2399
2400         op.u.ecc.function = cmd->u.ecc.function;
2401
2402         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2403         if (ret) {
2404                 cmd->engine_error = cmd_q->cmd_error;
2405                 goto e_dst;
2406         }
2407
2408         ecc->ecc_result = le16_to_cpup(
2409                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2410         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2411                 ret = -EIO;
2412                 goto e_dst;
2413         }
2414
2415         /* Save the workarea address since it is updated as we walk through
2416          * to copy the point math result
2417          */
2418         save = dst.address;
2419
2420         /* Save the ECC result X and Y coordinates */
2421         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2422                                 CCP_ECC_MODULUS_BYTES);
2423         dst.address += CCP_ECC_OUTPUT_SIZE;
2424         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2425                                 CCP_ECC_MODULUS_BYTES);
2426         dst.address += CCP_ECC_OUTPUT_SIZE;
2427
2428         /* Restore the workarea address */
2429         dst.address = save;
2430
2431 e_dst:
2432         ccp_dm_free(&dst);
2433
2434 e_src:
2435         ccp_dm_free(&src);
2436
2437         return ret;
2438 }
2439
2440 static noinline_for_stack int
2441 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2442 {
2443         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2444
2445         ecc->ecc_result = 0;
2446
2447         if (!ecc->mod ||
2448             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2449                 return -EINVAL;
2450
2451         switch (ecc->function) {
2452         case CCP_ECC_FUNCTION_MMUL_384BIT:
2453         case CCP_ECC_FUNCTION_MADD_384BIT:
2454         case CCP_ECC_FUNCTION_MINV_384BIT:
2455                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2456
2457         case CCP_ECC_FUNCTION_PADD_384BIT:
2458         case CCP_ECC_FUNCTION_PMUL_384BIT:
2459         case CCP_ECC_FUNCTION_PDBL_384BIT:
2460                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2461
2462         default:
2463                 return -EINVAL;
2464         }
2465 }
2466
2467 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2468 {
2469         int ret;
2470
2471         cmd->engine_error = 0;
2472         cmd_q->cmd_error = 0;
2473         cmd_q->int_rcvd = 0;
2474         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2475
2476         switch (cmd->engine) {
2477         case CCP_ENGINE_AES:
2478                 switch (cmd->u.aes.mode) {
2479                 case CCP_AES_MODE_CMAC:
2480                         ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2481                         break;
2482                 case CCP_AES_MODE_GCM:
2483                         ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2484                         break;
2485                 default:
2486                         ret = ccp_run_aes_cmd(cmd_q, cmd);
2487                         break;
2488                 }
2489                 break;
2490         case CCP_ENGINE_XTS_AES_128:
2491                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2492                 break;
2493         case CCP_ENGINE_DES3:
2494                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2495                 break;
2496         case CCP_ENGINE_SHA:
2497                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2498                 break;
2499         case CCP_ENGINE_RSA:
2500                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2501                 break;
2502         case CCP_ENGINE_PASSTHRU:
2503                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2504                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2505                 else
2506                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2507                 break;
2508         case CCP_ENGINE_ECC:
2509                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2510                 break;
2511         default:
2512                 ret = -EINVAL;
2513         }
2514
2515         return ret;
2516 }