drivers/crypto/ccp/ccp-ops.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8  * Author: Gary R Hook <gary.hook@amd.com>
   9  */
  10
  11 #include <linux/dma-mapping.h>
  12 #include <linux/module.h>
  13 #include <linux/kernel.h>
  14 #include <linux/interrupt.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/des.h>
  17 #include <linux/ccp.h>
  18
  19 #include "ccp-dev.h"
  20
  21 /* SHA initial context values */
  22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25         cpu_to_be32(SHA1_H4),
  26 };
  27
  28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  29         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  30         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  31         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  32         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  33 };
  34
  35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  36         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  37         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  38         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  39         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  40 };
  41
  42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  43         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  44         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  45         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  46         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  47 };
  48
  49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  50         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  51         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  52         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  53         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  54 };
  55
  56 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  57                                         ccp_gen_jobid(ccp) : 0)
  58
  59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  60 {
  61         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  62 }
  63
  64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  65 {
  66         if (wa->dma_count)
  67                 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  68
  69         wa->dma_count = 0;
  70 }
  71
  72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  73                                 struct scatterlist *sg, u64 len,
  74                                 enum dma_data_direction dma_dir)
  75 {
  76         memset(wa, 0, sizeof(*wa));
  77
  78         wa->sg = sg;
  79         if (!sg)
  80                 return 0;
  81
  82         wa->nents = sg_nents_for_len(sg, len);
  83         if (wa->nents < 0)
  84                 return wa->nents;
  85
  86         wa->bytes_left = len;
  87         wa->sg_used = 0;
  88
  89         if (len == 0)
  90                 return 0;
  91
  92         if (dma_dir == DMA_NONE)
  93                 return 0;
  94
  95         wa->dma_sg = sg;
  96         wa->dma_sg_head = sg;
  97         wa->dma_dev = dev;
  98         wa->dma_dir = dma_dir;
  99         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 100         if (!wa->dma_count)
 101                 return -ENOMEM;
 102
 103         return 0;
 104 }
 105
 106 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 107 {
 108         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 109         unsigned int sg_combined_len = 0;
 110
 111         if (!wa->sg)
 112                 return;
 113
 114         wa->sg_used += nbytes;
 115         wa->bytes_left -= nbytes;
 116         if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
 117                 /* Advance to the next DMA scatterlist entry */
 118                 wa->dma_sg = sg_next(wa->dma_sg);
 119
 120                 /* In the case that the DMA mapped scatterlist has entries
 121                  * that have been merged, the non-DMA mapped scatterlist
 122                  * must be advanced multiple times for each merged entry.
 123                  * This ensures that the current non-DMA mapped entry
 124                  * corresponds to the current DMA mapped entry.
 125                  */
 126                 do {
 127                         sg_combined_len += wa->sg->length;
 128                         wa->sg = sg_next(wa->sg);
 129                 } while (wa->sg_used > sg_combined_len);
 130
 131                 wa->sg_used = 0;
 132         }
 133 }
 134
 135 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 136 {
 137         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 138                 if (wa->address)
 139                         dma_pool_free(wa->dma_pool, wa->address,
 140                                       wa->dma.address);
 141         } else {
 142                 if (wa->dma.address)
 143                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 144                                          wa->dma.dir);
 145                 kfree(wa->address);
 146         }
 147
 148         wa->address = NULL;
 149         wa->dma.address = 0;
 150 }
 151
 152 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 153                                 struct ccp_cmd_queue *cmd_q,
 154                                 unsigned int len,
 155                                 enum dma_data_direction dir)
 156 {
 157         memset(wa, 0, sizeof(*wa));
 158
 159         if (!len)
 160                 return 0;
 161
 162         wa->dev = cmd_q->ccp->dev;
 163         wa->length = len;
 164
 165         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 166                 wa->dma_pool = cmd_q->dma_pool;
 167
 168                 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
 169                                              &wa->dma.address);
 170                 if (!wa->address)
 171                         return -ENOMEM;
 172
 173                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 174
 175         } else {
 176                 wa->address = kzalloc(len, GFP_KERNEL);
 177                 if (!wa->address)
 178                         return -ENOMEM;
 179
 180                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 181                                                  dir);
 182                 if (dma_mapping_error(wa->dev, wa->dma.address))
 183                         return -ENOMEM;
 184
 185                 wa->dma.length = len;
 186         }
 187         wa->dma.dir = dir;
 188
 189         return 0;
 190 }
 191
 192 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 193                            struct scatterlist *sg, unsigned int sg_offset,
 194                            unsigned int len)
 195 {
 196         WARN_ON(!wa->address);
 197
 198         if (len > (wa->length - wa_offset))
 199                 return -EINVAL;
 200
 201         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 202                                  0);
 203         return 0;
 204 }
 205
 206 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 207                             struct scatterlist *sg, unsigned int sg_offset,
 208                             unsigned int len)
 209 {
 210         WARN_ON(!wa->address);
 211
 212         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 213                                  1);
 214 }
 215
 216 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 217                                    unsigned int wa_offset,
 218                                    struct scatterlist *sg,
 219                                    unsigned int sg_offset,
 220                                    unsigned int len)
 221 {
 222         u8 *p, *q;
 223         int     rc;
 224
 225         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 226         if (rc)
 227                 return rc;
 228
 229         p = wa->address + wa_offset;
 230         q = p + len - 1;
 231         while (p < q) {
 232                 *p = *p ^ *q;
 233                 *q = *p ^ *q;
 234                 *p = *p ^ *q;
 235                 p++;
 236                 q--;
 237         }
 238         return 0;
 239 }
 240
 241 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 242                                     unsigned int wa_offset,
 243                                     struct scatterlist *sg,
 244                                     unsigned int sg_offset,
 245                                     unsigned int len)
 246 {
 247         u8 *p, *q;
 248
 249         p = wa->address + wa_offset;
 250         q = p + len - 1;
 251         while (p < q) {
 252                 *p = *p ^ *q;
 253                 *q = *p ^ *q;
 254                 *p = *p ^ *q;
 255                 p++;
 256                 q--;
 257         }
 258
 259         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 260 }
 261
 262 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 263 {
 264         ccp_dm_free(&data->dm_wa);
 265         ccp_sg_free(&data->sg_wa);
 266 }
 267
 268 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 269                          struct scatterlist *sg, u64 sg_len,
 270                          unsigned int dm_len,
 271                          enum dma_data_direction dir)
 272 {
 273         int ret;
 274
 275         memset(data, 0, sizeof(*data));
 276
 277         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 278                                    dir);
 279         if (ret)
 280                 goto e_err;
 281
 282         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 283         if (ret)
 284                 goto e_err;
 285
 286         return 0;
 287
 288 e_err:
 289         ccp_free_data(data, cmd_q);
 290
 291         return ret;
 292 }
 293
 294 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 295 {
 296         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 297         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 298         unsigned int buf_count, nbytes;
 299
 300         /* Clear the buffer if setting it */
 301         if (!from)
 302                 memset(dm_wa->address, 0, dm_wa->length);
 303
 304         if (!sg_wa->sg)
 305                 return 0;
 306
 307         /* Perform the copy operation
 308          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 309          *   an unsigned int
 310          */
 311         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 312         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 313                                  nbytes, from);
 314
 315         /* Update the structures and generate the count */
 316         buf_count = 0;
 317         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 318                 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
 319                              dm_wa->length - buf_count);
 320                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 321
 322                 buf_count += nbytes;
 323                 ccp_update_sg_workarea(sg_wa, nbytes);
 324         }
 325
 326         return buf_count;
 327 }
 328
 329 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 330 {
 331         return ccp_queue_buf(data, 0);
 332 }
 333
 334 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 335 {
 336         return ccp_queue_buf(data, 1);
 337 }
 338
 339 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 340                              struct ccp_op *op, unsigned int block_size,
 341                              bool blocksize_op)
 342 {
 343         unsigned int sg_src_len, sg_dst_len, op_len;
 344
 345         /* The CCP can only DMA from/to one address each per operation. This
 346          * requires that we find the smallest DMA area between the source
 347          * and destination. The resulting len values will always be <= UINT_MAX
 348          * because the dma length is an unsigned int.
 349          */
 350         sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
 351         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 352
 353         if (dst) {
 354                 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
 355                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 356                 op_len = min(sg_src_len, sg_dst_len);
 357         } else {
 358                 op_len = sg_src_len;
 359         }
 360
 361         /* The data operation length will be at least block_size in length
 362          * or the smaller of available sg room remaining for the source or
 363          * the destination
 364          */
 365         op_len = max(op_len, block_size);
 366
 367         /* Unless we have to buffer data, there's no reason to wait */
 368         op->soc = 0;
 369
 370         if (sg_src_len < block_size) {
 371                 /* Not enough data in the sg element, so it
 372                  * needs to be buffered into a blocksize chunk
 373                  */
 374                 int cp_len = ccp_fill_queue_buf(src);
 375
 376                 op->soc = 1;
 377                 op->src.u.dma.address = src->dm_wa.dma.address;
 378                 op->src.u.dma.offset = 0;
 379                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 380         } else {
 381                 /* Enough data in the sg element, but we need to
 382                  * adjust for any previously copied data
 383                  */
 384                 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
 385                 op->src.u.dma.offset = src->sg_wa.sg_used;
 386                 op->src.u.dma.length = op_len & ~(block_size - 1);
 387
 388                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 389         }
 390
 391         if (dst) {
 392                 if (sg_dst_len < block_size) {
 393                         /* Not enough room in the sg element or we're on the
 394                          * last piece of data (when using padding), so the
 395                          * output needs to be buffered into a blocksize chunk
 396                          */
 397                         op->soc = 1;
 398                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 399                         op->dst.u.dma.offset = 0;
 400                         op->dst.u.dma.length = op->src.u.dma.length;
 401                 } else {
 402                         /* Enough room in the sg element, but we need to
 403                          * adjust for any previously used area
 404                          */
 405                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
 406                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 407                         op->dst.u.dma.length = op->src.u.dma.length;
 408                 }
 409         }
 410 }
 411
 412 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 413                              struct ccp_op *op)
 414 {
 415         op->init = 0;
 416
 417         if (dst) {
 418                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 419                         ccp_empty_queue_buf(dst);
 420                 else
 421                         ccp_update_sg_workarea(&dst->sg_wa,
 422                                                op->dst.u.dma.length);
 423         }
 424 }
 425
 426 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 427                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 428                                u32 byte_swap, bool from)
 429 {
 430         struct ccp_op op;
 431
 432         memset(&op, 0, sizeof(op));
 433
 434         op.cmd_q = cmd_q;
 435         op.jobid = jobid;
 436         op.eom = 1;
 437
 438         if (from) {
 439                 op.soc = 1;
 440                 op.src.type = CCP_MEMTYPE_SB;
 441                 op.src.u.sb = sb;
 442                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 443                 op.dst.u.dma.address = wa->dma.address;
 444                 op.dst.u.dma.length = wa->length;
 445         } else {
 446                 op.src.type = CCP_MEMTYPE_SYSTEM;
 447                 op.src.u.dma.address = wa->dma.address;
 448                 op.src.u.dma.length = wa->length;
 449                 op.dst.type = CCP_MEMTYPE_SB;
 450                 op.dst.u.sb = sb;
 451         }
 452
 453         op.u.passthru.byte_swap = byte_swap;
 454
 455         return cmd_q->ccp->vdata->perform->passthru(&op);
 456 }
 457
 458 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 459                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 460                           u32 byte_swap)
 461 {
 462         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 463 }
 464
 465 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 466                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 467                             u32 byte_swap)
 468 {
 469         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 470 }
 471
 472 static noinline_for_stack int
 473 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 474 {
 475         struct ccp_aes_engine *aes = &cmd->u.aes;
 476         struct ccp_dm_workarea key, ctx;
 477         struct ccp_data src;
 478         struct ccp_op op;
 479         unsigned int dm_offset;
 480         int ret;
 481
 482         if (!((aes->key_len == AES_KEYSIZE_128) ||
 483               (aes->key_len == AES_KEYSIZE_192) ||
 484               (aes->key_len == AES_KEYSIZE_256)))
 485                 return -EINVAL;
 486
 487         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 488                 return -EINVAL;
 489
 490         if (aes->iv_len != AES_BLOCK_SIZE)
 491                 return -EINVAL;
 492
 493         if (!aes->key || !aes->iv || !aes->src)
 494                 return -EINVAL;
 495
 496         if (aes->cmac_final) {
 497                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 498                         return -EINVAL;
 499
 500                 if (!aes->cmac_key)
 501                         return -EINVAL;
 502         }
 503
 504         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 505         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 506
 507         ret = -EIO;
 508         memset(&op, 0, sizeof(op));
 509         op.cmd_q = cmd_q;
 510         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 511         op.sb_key = cmd_q->sb_key;
 512         op.sb_ctx = cmd_q->sb_ctx;
 513         op.init = 1;
 514         op.u.aes.type = aes->type;
 515         op.u.aes.mode = aes->mode;
 516         op.u.aes.action = aes->action;
 517
 518         /* All supported key sizes fit in a single (32-byte) SB entry
 519          * and must be in little endian format. Use the 256-bit byte
 520          * swap passthru option to convert from big endian to little
 521          * endian.
 522          */
 523         ret = ccp_init_dm_workarea(&key, cmd_q,
 524                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 525                                    DMA_TO_DEVICE);
 526         if (ret)
 527                 return ret;
 528
 529         dm_offset = CCP_SB_BYTES - aes->key_len;
 530         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 531         if (ret)
 532                 goto e_key;
 533         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 534                              CCP_PASSTHRU_BYTESWAP_256BIT);
 535         if (ret) {
 536                 cmd->engine_error = cmd_q->cmd_error;
 537                 goto e_key;
 538         }
 539
 540         /* The AES context fits in a single (32-byte) SB entry and
 541          * must be in little endian format. Use the 256-bit byte swap
 542          * passthru option to convert from big endian to little endian.
 543          */
 544         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 545                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 546                                    DMA_BIDIRECTIONAL);
 547         if (ret)
 548                 goto e_key;
 549
 550         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 551         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 552         if (ret)
 553                 goto e_ctx;
 554         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 555                              CCP_PASSTHRU_BYTESWAP_256BIT);
 556         if (ret) {
 557                 cmd->engine_error = cmd_q->cmd_error;
 558                 goto e_ctx;
 559         }
 560
 561         /* Send data to the CCP AES engine */
 562         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 563                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 564         if (ret)
 565                 goto e_ctx;
 566
 567         while (src.sg_wa.bytes_left) {
 568                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 569                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 570                         op.eom = 1;
 571
 572                         /* Push the K1/K2 key to the CCP now */
 573                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 574                                                op.sb_ctx,
 575                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 576                         if (ret) {
 577                                 cmd->engine_error = cmd_q->cmd_error;
 578                                 goto e_src;
 579                         }
 580
 581                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 582                                               aes->cmac_key_len);
 583                         if (ret)
 584                                 goto e_src;
 585                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 586                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 587                         if (ret) {
 588                                 cmd->engine_error = cmd_q->cmd_error;
 589                                 goto e_src;
 590                         }
 591                 }
 592
 593                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 594                 if (ret) {
 595                         cmd->engine_error = cmd_q->cmd_error;
 596                         goto e_src;
 597                 }
 598
 599                 ccp_process_data(&src, NULL, &op);
 600         }
 601
 602         /* Retrieve the AES context - convert from LE to BE using
 603          * 32-byte (256-bit) byteswapping
 604          */
 605         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 606                                CCP_PASSTHRU_BYTESWAP_256BIT);
 607         if (ret) {
 608                 cmd->engine_error = cmd_q->cmd_error;
 609                 goto e_src;
 610         }
 611
 612         /* ...but we only need AES_BLOCK_SIZE bytes */
 613         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 614         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 615
 616 e_src:
 617         ccp_free_data(&src, cmd_q);
 618
 619 e_ctx:
 620         ccp_dm_free(&ctx);
 621
 622 e_key:
 623         ccp_dm_free(&key);
 624
 625         return ret;
 626 }
 627
 628 static noinline_for_stack int
 629 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 630 {
 631         struct ccp_aes_engine *aes = &cmd->u.aes;
 632         struct ccp_dm_workarea key, ctx, final_wa, tag;
 633         struct ccp_data src, dst;
 634         struct ccp_data aad;
 635         struct ccp_op op;
 636         unsigned int dm_offset;
 637         unsigned int authsize;
 638         unsigned int jobid;
 639         unsigned int ilen;
 640         bool in_place = true; /* Default value */
 641         __be64 *final;
 642         int ret;
 643
 644         struct scatterlist *p_inp, sg_inp[2];
 645         struct scatterlist *p_tag, sg_tag[2];
 646         struct scatterlist *p_outp, sg_outp[2];
 647         struct scatterlist *p_aad;
 648
 649         if (!aes->iv)
 650                 return -EINVAL;
 651
 652         if (!((aes->key_len == AES_KEYSIZE_128) ||
 653                 (aes->key_len == AES_KEYSIZE_192) ||
 654                 (aes->key_len == AES_KEYSIZE_256)))
 655                 return -EINVAL;
 656
 657         if (!aes->key) /* Gotta have a key SGL */
 658                 return -EINVAL;
 659
 660         /* Zero defaults to 16 bytes, the maximum size */
 661         authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 662         switch (authsize) {
 663         case 16:
 664         case 15:
 665         case 14:
 666         case 13:
 667         case 12:
 668         case 8:
 669         case 4:
 670                 break;
 671         default:
 672                 return -EINVAL;
 673         }
 674
 675         /* First, decompose the source buffer into AAD & PT,
 676          * and the destination buffer into AAD, CT & tag, or
 677          * the input into CT & tag.
 678          * It is expected that the input and output SGs will
 679          * be valid, even if the AAD and input lengths are 0.
 680          */
 681         p_aad = aes->src;
 682         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 683         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 684         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 685                 ilen = aes->src_len;
 686                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 687         } else {
 688                 /* Input length for decryption includes tag */
 689                 ilen = aes->src_len - authsize;
 690                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 691         }
 692
 693         jobid = CCP_NEW_JOBID(cmd_q->ccp);
 694
 695         memset(&op, 0, sizeof(op));
 696         op.cmd_q = cmd_q;
 697         op.jobid = jobid;
 698         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 699         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 700         op.init = 1;
 701         op.u.aes.type = aes->type;
 702
 703         /* Copy the key to the LSB */
 704         ret = ccp_init_dm_workarea(&key, cmd_q,
 705                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 706                                    DMA_TO_DEVICE);
 707         if (ret)
 708                 return ret;
 709
 710         dm_offset = CCP_SB_BYTES - aes->key_len;
 711         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 712         if (ret)
 713                 goto e_key;
 714         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 715                              CCP_PASSTHRU_BYTESWAP_256BIT);
 716         if (ret) {
 717                 cmd->engine_error = cmd_q->cmd_error;
 718                 goto e_key;
 719         }
 720
 721         /* Copy the context (IV) to the LSB.
 722          * There is an assumption here that the IV is 96 bits in length, plus
 723          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 724          */
 725         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 726                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 727                                    DMA_BIDIRECTIONAL);
 728         if (ret)
 729                 goto e_key;
 730
 731         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 732         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 733         if (ret)
 734                 goto e_ctx;
 735
 736         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 737                              CCP_PASSTHRU_BYTESWAP_256BIT);
 738         if (ret) {
 739                 cmd->engine_error = cmd_q->cmd_error;
 740                 goto e_ctx;
 741         }
 742
 743         op.init = 1;
 744         if (aes->aad_len > 0) {
 745                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 746                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 747                                     AES_BLOCK_SIZE,
 748                                     DMA_TO_DEVICE);
 749                 if (ret)
 750                         goto e_ctx;
 751
 752                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 753                 op.u.aes.action = CCP_AES_GHASHAAD;
 754
 755                 while (aad.sg_wa.bytes_left) {
 756                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 757
 758                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 759                         if (ret) {
 760                                 cmd->engine_error = cmd_q->cmd_error;
 761                                 goto e_aad;
 762                         }
 763
 764                         ccp_process_data(&aad, NULL, &op);
 765                         op.init = 0;
 766                 }
 767         }
 768
 769         op.u.aes.mode = CCP_AES_MODE_GCTR;
 770         op.u.aes.action = aes->action;
 771
 772         if (ilen > 0) {
 773                 /* Step 2: Run a GCTR over the plaintext */
 774                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 775
 776                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 777                                     AES_BLOCK_SIZE,
 778                                     in_place ? DMA_BIDIRECTIONAL
 779                                              : DMA_TO_DEVICE);
 780                 if (ret)
 781                         goto e_aad;
 782
 783                 if (in_place) {
 784                         dst = src;
 785                 } else {
 786                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 787                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 788                         if (ret)
 789                                 goto e_src;
 790                 }
 791
 792                 op.soc = 0;
 793                 op.eom = 0;
 794                 op.init = 1;
 795                 while (src.sg_wa.bytes_left) {
 796                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 797                         if (!src.sg_wa.bytes_left) {
 798                                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 799
 800                                 if (nbytes) {
 801                                         op.eom = 1;
 802                                         op.u.aes.size = (nbytes * 8) - 1;
 803                                 }
 804                         }
 805
 806                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 807                         if (ret) {
 808                                 cmd->engine_error = cmd_q->cmd_error;
 809                                 goto e_dst;
 810                         }
 811
 812                         ccp_process_data(&src, &dst, &op);
 813                         op.init = 0;
 814                 }
 815         }
 816
 817         /* Step 3: Update the IV portion of the context with the original IV */
 818         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 819                                CCP_PASSTHRU_BYTESWAP_256BIT);
 820         if (ret) {
 821                 cmd->engine_error = cmd_q->cmd_error;
 822                 goto e_dst;
 823         }
 824
 825         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 826         if (ret)
 827                 goto e_dst;
 828
 829         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 830                              CCP_PASSTHRU_BYTESWAP_256BIT);
 831         if (ret) {
 832                 cmd->engine_error = cmd_q->cmd_error;
 833                 goto e_dst;
 834         }
 835
 836         /* Step 4: Concatenate the lengths of the AAD and source, and
 837          * hash that 16 byte buffer.
 838          */
 839         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 840                                    DMA_BIDIRECTIONAL);
 841         if (ret)
 842                 goto e_dst;
 843         final = (__be64 *)final_wa.address;
 844         final[0] = cpu_to_be64(aes->aad_len * 8);
 845         final[1] = cpu_to_be64(ilen * 8);
 846
 847         memset(&op, 0, sizeof(op));
 848         op.cmd_q = cmd_q;
 849         op.jobid = jobid;
 850         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 851         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 852         op.init = 1;
 853         op.u.aes.type = aes->type;
 854         op.u.aes.mode = CCP_AES_MODE_GHASH;
 855         op.u.aes.action = CCP_AES_GHASHFINAL;
 856         op.src.type = CCP_MEMTYPE_SYSTEM;
 857         op.src.u.dma.address = final_wa.dma.address;
 858         op.src.u.dma.length = AES_BLOCK_SIZE;
 859         op.dst.type = CCP_MEMTYPE_SYSTEM;
 860         op.dst.u.dma.address = final_wa.dma.address;
 861         op.dst.u.dma.length = AES_BLOCK_SIZE;
 862         op.eom = 1;
 863         op.u.aes.size = 0;
 864         ret = cmd_q->ccp->vdata->perform->aes(&op);
 865         if (ret)
 866                 goto e_final_wa;
 867
 868         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 869                 /* Put the ciphered tag after the ciphertext. */
 870                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 871         } else {
 872                 /* Does this ciphered tag match the input? */
 873                 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 874                                            DMA_BIDIRECTIONAL);
 875                 if (ret)
 876                         goto e_final_wa;
 877                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 878                 if (ret) {
 879                         ccp_dm_free(&tag);
 880                         goto e_final_wa;
 881                 }
 882
 883                 ret = crypto_memneq(tag.address, final_wa.address,
 884                                     authsize) ? -EBADMSG : 0;
 885                 ccp_dm_free(&tag);
 886         }
 887
 888 e_final_wa:
 889         ccp_dm_free(&final_wa);
 890
 891 e_dst:
 892         if (ilen > 0 && !in_place)
 893                 ccp_free_data(&dst, cmd_q);
 894
 895 e_src:
 896         if (ilen > 0)
 897                 ccp_free_data(&src, cmd_q);
 898
 899 e_aad:
 900         if (aes->aad_len)
 901                 ccp_free_data(&aad, cmd_q);
 902
 903 e_ctx:
 904         ccp_dm_free(&ctx);
 905
 906 e_key:
 907         ccp_dm_free(&key);
 908
 909         return ret;
 910 }
 911
 912 static noinline_for_stack int
 913 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 914 {
 915         struct ccp_aes_engine *aes = &cmd->u.aes;
 916         struct ccp_dm_workarea key, ctx;
 917         struct ccp_data src, dst;
 918         struct ccp_op op;
 919         unsigned int dm_offset;
 920         bool in_place = false;
 921         int ret;
 922
 923         if (!((aes->key_len == AES_KEYSIZE_128) ||
 924               (aes->key_len == AES_KEYSIZE_192) ||
 925               (aes->key_len == AES_KEYSIZE_256)))
 926                 return -EINVAL;
 927
 928         if (((aes->mode == CCP_AES_MODE_ECB) ||
 929              (aes->mode == CCP_AES_MODE_CBC)) &&
 930             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 931                 return -EINVAL;
 932
 933         if (!aes->key || !aes->src || !aes->dst)
 934                 return -EINVAL;
 935
 936         if (aes->mode != CCP_AES_MODE_ECB) {
 937                 if (aes->iv_len != AES_BLOCK_SIZE)
 938                         return -EINVAL;
 939
 940                 if (!aes->iv)
 941                         return -EINVAL;
 942         }
 943
 944         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 945         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 946
 947         ret = -EIO;
 948         memset(&op, 0, sizeof(op));
 949         op.cmd_q = cmd_q;
 950         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 951         op.sb_key = cmd_q->sb_key;
 952         op.sb_ctx = cmd_q->sb_ctx;
 953         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 954         op.u.aes.type = aes->type;
 955         op.u.aes.mode = aes->mode;
 956         op.u.aes.action = aes->action;
 957
 958         /* All supported key sizes fit in a single (32-byte) SB entry
 959          * and must be in little endian format. Use the 256-bit byte
 960          * swap passthru option to convert from big endian to little
 961          * endian.
 962          */
 963         ret = ccp_init_dm_workarea(&key, cmd_q,
 964                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 965                                    DMA_TO_DEVICE);
 966         if (ret)
 967                 return ret;
 968
 969         dm_offset = CCP_SB_BYTES - aes->key_len;
 970         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 971         if (ret)
 972                 goto e_key;
 973         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 974                              CCP_PASSTHRU_BYTESWAP_256BIT);
 975         if (ret) {
 976                 cmd->engine_error = cmd_q->cmd_error;
 977                 goto e_key;
 978         }
 979
 980         /* The AES context fits in a single (32-byte) SB entry and
 981          * must be in little endian format. Use the 256-bit byte swap
 982          * passthru option to convert from big endian to little endian.
 983          */
 984         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 985                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 986                                    DMA_BIDIRECTIONAL);
 987         if (ret)
 988                 goto e_key;
 989
 990         if (aes->mode != CCP_AES_MODE_ECB) {
 991                 /* Load the AES context - convert to LE */
 992                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 993                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 994                 if (ret)
 995                         goto e_ctx;
 996                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 997                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 998                 if (ret) {
 999                         cmd->engine_error = cmd_q->cmd_error;
1000                         goto e_ctx;
1001                 }
1002         }
1003         switch (aes->mode) {
1004         case CCP_AES_MODE_CFB: /* CFB128 only */
1005         case CCP_AES_MODE_CTR:
1006                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1007                 break;
1008         default:
1009                 op.u.aes.size = 0;
1010         }
1011
1012         /* Prepare the input and output data workareas. For in-place
1013          * operations we need to set the dma direction to BIDIRECTIONAL
1014          * and copy the src workarea to the dst workarea.
1015          */
1016         if (sg_virt(aes->src) == sg_virt(aes->dst))
1017                 in_place = true;
1018
1019         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1020                             AES_BLOCK_SIZE,
1021                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1022         if (ret)
1023                 goto e_ctx;
1024
1025         if (in_place) {
1026                 dst = src;
1027         } else {
1028                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1029                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1030                 if (ret)
1031                         goto e_src;
1032         }
1033
1034         /* Send data to the CCP AES engine */
1035         while (src.sg_wa.bytes_left) {
1036                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1037                 if (!src.sg_wa.bytes_left) {
1038                         op.eom = 1;
1039
1040                         /* Since we don't retrieve the AES context in ECB
1041                          * mode we have to wait for the operation to complete
1042                          * on the last piece of data
1043                          */
1044                         if (aes->mode == CCP_AES_MODE_ECB)
1045                                 op.soc = 1;
1046                 }
1047
1048                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1049                 if (ret) {
1050                         cmd->engine_error = cmd_q->cmd_error;
1051                         goto e_dst;
1052                 }
1053
1054                 ccp_process_data(&src, &dst, &op);
1055         }
1056
1057         if (aes->mode != CCP_AES_MODE_ECB) {
1058                 /* Retrieve the AES context - convert from LE to BE using
1059                  * 32-byte (256-bit) byteswapping
1060                  */
1061                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1062                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1063                 if (ret) {
1064                         cmd->engine_error = cmd_q->cmd_error;
1065                         goto e_dst;
1066                 }
1067
1068                 /* ...but we only need AES_BLOCK_SIZE bytes */
1069                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1070                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1071         }
1072
1073 e_dst:
1074         if (!in_place)
1075                 ccp_free_data(&dst, cmd_q);
1076
1077 e_src:
1078         ccp_free_data(&src, cmd_q);
1079
1080 e_ctx:
1081         ccp_dm_free(&ctx);
1082
1083 e_key:
1084         ccp_dm_free(&key);
1085
1086         return ret;
1087 }
1088
1089 static noinline_for_stack int
1090 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1091 {
1092         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1093         struct ccp_dm_workarea key, ctx;
1094         struct ccp_data src, dst;
1095         struct ccp_op op;
1096         unsigned int unit_size, dm_offset;
1097         bool in_place = false;
1098         unsigned int sb_count;
1099         enum ccp_aes_type aestype;
1100         int ret;
1101
1102         switch (xts->unit_size) {
1103         case CCP_XTS_AES_UNIT_SIZE_16:
1104                 unit_size = 16;
1105                 break;
1106         case CCP_XTS_AES_UNIT_SIZE_512:
1107                 unit_size = 512;
1108                 break;
1109         case CCP_XTS_AES_UNIT_SIZE_1024:
1110                 unit_size = 1024;
1111                 break;
1112         case CCP_XTS_AES_UNIT_SIZE_2048:
1113                 unit_size = 2048;
1114                 break;
1115         case CCP_XTS_AES_UNIT_SIZE_4096:
1116                 unit_size = 4096;
1117                 break;
1118
1119         default:
1120                 return -EINVAL;
1121         }
1122
1123         if (xts->key_len == AES_KEYSIZE_128)
1124                 aestype = CCP_AES_TYPE_128;
1125         else if (xts->key_len == AES_KEYSIZE_256)
1126                 aestype = CCP_AES_TYPE_256;
1127         else
1128                 return -EINVAL;
1129
1130         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1131                 return -EINVAL;
1132
1133         if (xts->iv_len != AES_BLOCK_SIZE)
1134                 return -EINVAL;
1135
1136         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1137                 return -EINVAL;
1138
1139         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1140         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1141
1142         ret = -EIO;
1143         memset(&op, 0, sizeof(op));
1144         op.cmd_q = cmd_q;
1145         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1146         op.sb_key = cmd_q->sb_key;
1147         op.sb_ctx = cmd_q->sb_ctx;
1148         op.init = 1;
1149         op.u.xts.type = aestype;
1150         op.u.xts.action = xts->action;
1151         op.u.xts.unit_size = xts->unit_size;
1152
1153         /* A version 3 device only supports 128-bit keys, which fits into a
1154          * single SB entry. A version 5 device uses a 512-bit vector, so two
1155          * SB entries.
1156          */
1157         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1158                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1159         else
1160                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1161         ret = ccp_init_dm_workarea(&key, cmd_q,
1162                                    sb_count * CCP_SB_BYTES,
1163                                    DMA_TO_DEVICE);
1164         if (ret)
1165                 return ret;
1166
1167         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1168                 /* All supported key sizes must be in little endian format.
1169                  * Use the 256-bit byte swap passthru option to convert from
1170                  * big endian to little endian.
1171                  */
1172                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1173                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1174                 if (ret)
1175                         goto e_key;
1176                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1177                 if (ret)
1178                         goto e_key;
1179         } else {
1180                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1181                  * occupies 256 bits, or one entire slot, and is zero-padded.
1182                  */
1183                 unsigned int pad;
1184
1185                 dm_offset = CCP_SB_BYTES;
1186                 pad = dm_offset - xts->key_len;
1187                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1188                 if (ret)
1189                         goto e_key;
1190                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1191                                       xts->key_len, xts->key_len);
1192                 if (ret)
1193                         goto e_key;
1194         }
1195         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1196                              CCP_PASSTHRU_BYTESWAP_256BIT);
1197         if (ret) {
1198                 cmd->engine_error = cmd_q->cmd_error;
1199                 goto e_key;
1200         }
1201
1202         /* The AES context fits in a single (32-byte) SB entry and
1203          * for XTS is already in little endian format so no byte swapping
1204          * is needed.
1205          */
1206         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1207                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1208                                    DMA_BIDIRECTIONAL);
1209         if (ret)
1210                 goto e_key;
1211
1212         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1213         if (ret)
1214                 goto e_ctx;
1215         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1216                              CCP_PASSTHRU_BYTESWAP_NOOP);
1217         if (ret) {
1218                 cmd->engine_error = cmd_q->cmd_error;
1219                 goto e_ctx;
1220         }
1221
1222         /* Prepare the input and output data workareas. For in-place
1223          * operations we need to set the dma direction to BIDIRECTIONAL
1224          * and copy the src workarea to the dst workarea.
1225          */
1226         if (sg_virt(xts->src) == sg_virt(xts->dst))
1227                 in_place = true;
1228
1229         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1230                             unit_size,
1231                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1232         if (ret)
1233                 goto e_ctx;
1234
1235         if (in_place) {
1236                 dst = src;
1237         } else {
1238                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1239                                     unit_size, DMA_FROM_DEVICE);
1240                 if (ret)
1241                         goto e_src;
1242         }
1243
1244         /* Send data to the CCP AES engine */
1245         while (src.sg_wa.bytes_left) {
1246                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1247                 if (!src.sg_wa.bytes_left)
1248                         op.eom = 1;
1249
1250                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1251                 if (ret) {
1252                         cmd->engine_error = cmd_q->cmd_error;
1253                         goto e_dst;
1254                 }
1255
1256                 ccp_process_data(&src, &dst, &op);
1257         }
1258
1259         /* Retrieve the AES context - convert from LE to BE using
1260          * 32-byte (256-bit) byteswapping
1261          */
1262         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1263                                CCP_PASSTHRU_BYTESWAP_256BIT);
1264         if (ret) {
1265                 cmd->engine_error = cmd_q->cmd_error;
1266                 goto e_dst;
1267         }
1268
1269         /* ...but we only need AES_BLOCK_SIZE bytes */
1270         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1271         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1272
1273 e_dst:
1274         if (!in_place)
1275                 ccp_free_data(&dst, cmd_q);
1276
1277 e_src:
1278         ccp_free_data(&src, cmd_q);
1279
1280 e_ctx:
1281         ccp_dm_free(&ctx);
1282
1283 e_key:
1284         ccp_dm_free(&key);
1285
1286         return ret;
1287 }
1288
1289 static noinline_for_stack int
1290 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1291 {
1292         struct ccp_des3_engine *des3 = &cmd->u.des3;
1293
1294         struct ccp_dm_workarea key, ctx;
1295         struct ccp_data src, dst;
1296         struct ccp_op op;
1297         unsigned int dm_offset;
1298         unsigned int len_singlekey;
1299         bool in_place = false;
1300         int ret;
1301
1302         /* Error checks */
1303         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1304                 return -EINVAL;
1305
1306         if (!cmd_q->ccp->vdata->perform->des3)
1307                 return -EINVAL;
1308
1309         if (des3->key_len != DES3_EDE_KEY_SIZE)
1310                 return -EINVAL;
1311
1312         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1313                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1314                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1315                 return -EINVAL;
1316
1317         if (!des3->key || !des3->src || !des3->dst)
1318                 return -EINVAL;
1319
1320         if (des3->mode != CCP_DES3_MODE_ECB) {
1321                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1322                         return -EINVAL;
1323
1324                 if (!des3->iv)
1325                         return -EINVAL;
1326         }
1327
1328         /* Zero out all the fields of the command desc */
1329         memset(&op, 0, sizeof(op));
1330
1331         /* Set up the Function field */
1332         op.cmd_q = cmd_q;
1333         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1334         op.sb_key = cmd_q->sb_key;
1335
1336         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1337         op.u.des3.type = des3->type;
1338         op.u.des3.mode = des3->mode;
1339         op.u.des3.action = des3->action;
1340
1341         /*
1342          * All supported key sizes fit in a single (32-byte) KSB entry and
1343          * (like AES) must be in little endian format. Use the 256-bit byte
1344          * swap passthru option to convert from big endian to little endian.
1345          */
1346         ret = ccp_init_dm_workarea(&key, cmd_q,
1347                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1348                                    DMA_TO_DEVICE);
1349         if (ret)
1350                 return ret;
1351
1352         /*
1353          * The contents of the key triplet are in the reverse order of what
1354          * is required by the engine. Copy the 3 pieces individually to put
1355          * them where they belong.
1356          */
1357         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1358
1359         len_singlekey = des3->key_len / 3;
1360         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1361                               des3->key, 0, len_singlekey);
1362         if (ret)
1363                 goto e_key;
1364         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1365                               des3->key, len_singlekey, len_singlekey);
1366         if (ret)
1367                 goto e_key;
1368         ret = ccp_set_dm_area(&key, dm_offset,
1369                               des3->key, 2 * len_singlekey, len_singlekey);
1370         if (ret)
1371                 goto e_key;
1372
1373         /* Copy the key to the SB */
1374         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1375                              CCP_PASSTHRU_BYTESWAP_256BIT);
1376         if (ret) {
1377                 cmd->engine_error = cmd_q->cmd_error;
1378                 goto e_key;
1379         }
1380
1381         /*
1382          * The DES3 context fits in a single (32-byte) KSB entry and
1383          * must be in little endian format. Use the 256-bit byte swap
1384          * passthru option to convert from big endian to little endian.
1385          */
1386         if (des3->mode != CCP_DES3_MODE_ECB) {
1387                 op.sb_ctx = cmd_q->sb_ctx;
1388
1389                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1390                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1391                                            DMA_BIDIRECTIONAL);
1392                 if (ret)
1393                         goto e_key;
1394
1395                 /* Load the context into the LSB */
1396                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1397                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1398                                       des3->iv_len);
1399                 if (ret)
1400                         goto e_ctx;
1401
1402                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1403                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1404                 if (ret) {
1405                         cmd->engine_error = cmd_q->cmd_error;
1406                         goto e_ctx;
1407                 }
1408         }
1409
1410         /*
1411          * Prepare the input and output data workareas. For in-place
1412          * operations we need to set the dma direction to BIDIRECTIONAL
1413          * and copy the src workarea to the dst workarea.
1414          */
1415         if (sg_virt(des3->src) == sg_virt(des3->dst))
1416                 in_place = true;
1417
1418         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1419                         DES3_EDE_BLOCK_SIZE,
1420                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1421         if (ret)
1422                 goto e_ctx;
1423
1424         if (in_place)
1425                 dst = src;
1426         else {
1427                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1428                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1429                 if (ret)
1430                         goto e_src;
1431         }
1432
1433         /* Send data to the CCP DES3 engine */
1434         while (src.sg_wa.bytes_left) {
1435                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1436                 if (!src.sg_wa.bytes_left) {
1437                         op.eom = 1;
1438
1439                         /* Since we don't retrieve the context in ECB mode
1440                          * we have to wait for the operation to complete
1441                          * on the last piece of data
1442                          */
1443                         op.soc = 0;
1444                 }
1445
1446                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1447                 if (ret) {
1448                         cmd->engine_error = cmd_q->cmd_error;
1449                         goto e_dst;
1450                 }
1451
1452                 ccp_process_data(&src, &dst, &op);
1453         }
1454
1455         if (des3->mode != CCP_DES3_MODE_ECB) {
1456                 /* Retrieve the context and make BE */
1457                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1458                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1459                 if (ret) {
1460                         cmd->engine_error = cmd_q->cmd_error;
1461                         goto e_dst;
1462                 }
1463
1464                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1465                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1466                                 DES3_EDE_BLOCK_SIZE);
1467         }
1468 e_dst:
1469         if (!in_place)
1470                 ccp_free_data(&dst, cmd_q);
1471
1472 e_src:
1473         ccp_free_data(&src, cmd_q);
1474
1475 e_ctx:
1476         if (des3->mode != CCP_DES3_MODE_ECB)
1477                 ccp_dm_free(&ctx);
1478
1479 e_key:
1480         ccp_dm_free(&key);
1481
1482         return ret;
1483 }
1484
1485 static noinline_for_stack int
1486 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1487 {
1488         struct ccp_sha_engine *sha = &cmd->u.sha;
1489         struct ccp_dm_workarea ctx;
1490         struct ccp_data src;
1491         struct ccp_op op;
1492         unsigned int ioffset, ooffset;
1493         unsigned int digest_size;
1494         int sb_count;
1495         const void *init;
1496         u64 block_size;
1497         int ctx_size;
1498         int ret;
1499
1500         switch (sha->type) {
1501         case CCP_SHA_TYPE_1:
1502                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1503                         return -EINVAL;
1504                 block_size = SHA1_BLOCK_SIZE;
1505                 break;
1506         case CCP_SHA_TYPE_224:
1507                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1508                         return -EINVAL;
1509                 block_size = SHA224_BLOCK_SIZE;
1510                 break;
1511         case CCP_SHA_TYPE_256:
1512                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1513                         return -EINVAL;
1514                 block_size = SHA256_BLOCK_SIZE;
1515                 break;
1516         case CCP_SHA_TYPE_384:
1517                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1518                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1519                         return -EINVAL;
1520                 block_size = SHA384_BLOCK_SIZE;
1521                 break;
1522         case CCP_SHA_TYPE_512:
1523                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1524                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1525                         return -EINVAL;
1526                 block_size = SHA512_BLOCK_SIZE;
1527                 break;
1528         default:
1529                 return -EINVAL;
1530         }
1531
1532         if (!sha->ctx)
1533                 return -EINVAL;
1534
1535         if (!sha->final && (sha->src_len & (block_size - 1)))
1536                 return -EINVAL;
1537
1538         /* The version 3 device can't handle zero-length input */
1539         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1540
1541                 if (!sha->src_len) {
1542                         unsigned int digest_len;
1543                         const u8 *sha_zero;
1544
1545                         /* Not final, just return */
1546                         if (!sha->final)
1547                                 return 0;
1548
1549                         /* CCP can't do a zero length sha operation so the
1550                          * caller must buffer the data.
1551                          */
1552                         if (sha->msg_bits)
1553                                 return -EINVAL;
1554
1555                         /* The CCP cannot perform zero-length sha operations
1556                          * so the caller is required to buffer data for the
1557                          * final operation. However, a sha operation for a
1558                          * message with a total length of zero is valid so
1559                          * known values are required to supply the result.
1560                          */
1561                         switch (sha->type) {
1562                         case CCP_SHA_TYPE_1:
1563                                 sha_zero = sha1_zero_message_hash;
1564                                 digest_len = SHA1_DIGEST_SIZE;
1565                                 break;
1566                         case CCP_SHA_TYPE_224:
1567                                 sha_zero = sha224_zero_message_hash;
1568                                 digest_len = SHA224_DIGEST_SIZE;
1569                                 break;
1570                         case CCP_SHA_TYPE_256:
1571                                 sha_zero = sha256_zero_message_hash;
1572                                 digest_len = SHA256_DIGEST_SIZE;
1573                                 break;
1574                         default:
1575                                 return -EINVAL;
1576                         }
1577
1578                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1579                                                  digest_len, 1);
1580
1581                         return 0;
1582                 }
1583         }
1584
1585         /* Set variables used throughout */
1586         switch (sha->type) {
1587         case CCP_SHA_TYPE_1:
1588                 digest_size = SHA1_DIGEST_SIZE;
1589                 init = (void *) ccp_sha1_init;
1590                 ctx_size = SHA1_DIGEST_SIZE;
1591                 sb_count = 1;
1592                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1594                 else
1595                         ooffset = ioffset = 0;
1596                 break;
1597         case CCP_SHA_TYPE_224:
1598                 digest_size = SHA224_DIGEST_SIZE;
1599                 init = (void *) ccp_sha224_init;
1600                 ctx_size = SHA256_DIGEST_SIZE;
1601                 sb_count = 1;
1602                 ioffset = 0;
1603                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1604                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1605                 else
1606                         ooffset = 0;
1607                 break;
1608         case CCP_SHA_TYPE_256:
1609                 digest_size = SHA256_DIGEST_SIZE;
1610                 init = (void *) ccp_sha256_init;
1611                 ctx_size = SHA256_DIGEST_SIZE;
1612                 sb_count = 1;
1613                 ooffset = ioffset = 0;
1614                 break;
1615         case CCP_SHA_TYPE_384:
1616                 digest_size = SHA384_DIGEST_SIZE;
1617                 init = (void *) ccp_sha384_init;
1618                 ctx_size = SHA512_DIGEST_SIZE;
1619                 sb_count = 2;
1620                 ioffset = 0;
1621                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1622                 break;
1623         case CCP_SHA_TYPE_512:
1624                 digest_size = SHA512_DIGEST_SIZE;
1625                 init = (void *) ccp_sha512_init;
1626                 ctx_size = SHA512_DIGEST_SIZE;
1627                 sb_count = 2;
1628                 ooffset = ioffset = 0;
1629                 break;
1630         default:
1631                 ret = -EINVAL;
1632                 goto e_data;
1633         }
1634
1635         /* For zero-length plaintext the src pointer is ignored;
1636          * otherwise both parts must be valid
1637          */
1638         if (sha->src_len && !sha->src)
1639                 return -EINVAL;
1640
1641         memset(&op, 0, sizeof(op));
1642         op.cmd_q = cmd_q;
1643         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1644         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1645         op.u.sha.type = sha->type;
1646         op.u.sha.msg_bits = sha->msg_bits;
1647
1648         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1649          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1650          * first slot, and the left half in the second. Each portion must then
1651          * be in little endian format: use the 256-bit byte swap option.
1652          */
1653         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1654                                    DMA_BIDIRECTIONAL);
1655         if (ret)
1656                 return ret;
1657         if (sha->first) {
1658                 switch (sha->type) {
1659                 case CCP_SHA_TYPE_1:
1660                 case CCP_SHA_TYPE_224:
1661                 case CCP_SHA_TYPE_256:
1662                         memcpy(ctx.address + ioffset, init, ctx_size);
1663                         break;
1664                 case CCP_SHA_TYPE_384:
1665                 case CCP_SHA_TYPE_512:
1666                         memcpy(ctx.address + ctx_size / 2, init,
1667                                ctx_size / 2);
1668                         memcpy(ctx.address, init + ctx_size / 2,
1669                                ctx_size / 2);
1670                         break;
1671                 default:
1672                         ret = -EINVAL;
1673                         goto e_ctx;
1674                 }
1675         } else {
1676                 /* Restore the context */
1677                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1678                                       sb_count * CCP_SB_BYTES);
1679                 if (ret)
1680                         goto e_ctx;
1681         }
1682
1683         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1684                              CCP_PASSTHRU_BYTESWAP_256BIT);
1685         if (ret) {
1686                 cmd->engine_error = cmd_q->cmd_error;
1687                 goto e_ctx;
1688         }
1689
1690         if (sha->src) {
1691                 /* Send data to the CCP SHA engine; block_size is set above */
1692                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1693                                     block_size, DMA_TO_DEVICE);
1694                 if (ret)
1695                         goto e_ctx;
1696
1697                 while (src.sg_wa.bytes_left) {
1698                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1699                         if (sha->final && !src.sg_wa.bytes_left)
1700                                 op.eom = 1;
1701
1702                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1703                         if (ret) {
1704                                 cmd->engine_error = cmd_q->cmd_error;
1705                                 goto e_data;
1706                         }
1707
1708                         ccp_process_data(&src, NULL, &op);
1709                 }
1710         } else {
1711                 op.eom = 1;
1712                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1713                 if (ret) {
1714                         cmd->engine_error = cmd_q->cmd_error;
1715                         goto e_data;
1716                 }
1717         }
1718
1719         /* Retrieve the SHA context - convert from LE to BE using
1720          * 32-byte (256-bit) byteswapping to BE
1721          */
1722         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1723                                CCP_PASSTHRU_BYTESWAP_256BIT);
1724         if (ret) {
1725                 cmd->engine_error = cmd_q->cmd_error;
1726                 goto e_data;
1727         }
1728
1729         if (sha->final) {
1730                 /* Finishing up, so get the digest */
1731                 switch (sha->type) {
1732                 case CCP_SHA_TYPE_1:
1733                 case CCP_SHA_TYPE_224:
1734                 case CCP_SHA_TYPE_256:
1735                         ccp_get_dm_area(&ctx, ooffset,
1736                                         sha->ctx, 0,
1737                                         digest_size);
1738                         break;
1739                 case CCP_SHA_TYPE_384:
1740                 case CCP_SHA_TYPE_512:
1741                         ccp_get_dm_area(&ctx, 0,
1742                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1743                                         LSB_ITEM_SIZE);
1744                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1745                                         sha->ctx, 0,
1746                                         LSB_ITEM_SIZE - ooffset);
1747                         break;
1748                 default:
1749                         ret = -EINVAL;
1750                         goto e_data;
1751                 }
1752         } else {
1753                 /* Stash the context */
1754                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1755                                 sb_count * CCP_SB_BYTES);
1756         }
1757
1758         if (sha->final && sha->opad) {
1759                 /* HMAC operation, recursively perform final SHA */
1760                 struct ccp_cmd hmac_cmd;
1761                 struct scatterlist sg;
1762                 u8 *hmac_buf;
1763
1764                 if (sha->opad_len != block_size) {
1765                         ret = -EINVAL;
1766                         goto e_data;
1767                 }
1768
1769                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1770                 if (!hmac_buf) {
1771                         ret = -ENOMEM;
1772                         goto e_data;
1773                 }
1774                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1775
1776                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1777                 switch (sha->type) {
1778                 case CCP_SHA_TYPE_1:
1779                 case CCP_SHA_TYPE_224:
1780                 case CCP_SHA_TYPE_256:
1781                         memcpy(hmac_buf + block_size,
1782                                ctx.address + ooffset,
1783                                digest_size);
1784                         break;
1785                 case CCP_SHA_TYPE_384:
1786                 case CCP_SHA_TYPE_512:
1787                         memcpy(hmac_buf + block_size,
1788                                ctx.address + LSB_ITEM_SIZE + ooffset,
1789                                LSB_ITEM_SIZE);
1790                         memcpy(hmac_buf + block_size +
1791                                (LSB_ITEM_SIZE - ooffset),
1792                                ctx.address,
1793                                LSB_ITEM_SIZE);
1794                         break;
1795                 default:
1796                         kfree(hmac_buf);
1797                         ret = -EINVAL;
1798                         goto e_data;
1799                 }
1800
1801                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1802                 hmac_cmd.engine = CCP_ENGINE_SHA;
1803                 hmac_cmd.u.sha.type = sha->type;
1804                 hmac_cmd.u.sha.ctx = sha->ctx;
1805                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1806                 hmac_cmd.u.sha.src = &sg;
1807                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1808                 hmac_cmd.u.sha.opad = NULL;
1809                 hmac_cmd.u.sha.opad_len = 0;
1810                 hmac_cmd.u.sha.first = 1;
1811                 hmac_cmd.u.sha.final = 1;
1812                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1813
1814                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1815                 if (ret)
1816                         cmd->engine_error = hmac_cmd.engine_error;
1817
1818                 kfree(hmac_buf);
1819         }
1820
1821 e_data:
1822         if (sha->src)
1823                 ccp_free_data(&src, cmd_q);
1824
1825 e_ctx:
1826         ccp_dm_free(&ctx);
1827
1828         return ret;
1829 }
1830
1831 static noinline_for_stack int
1832 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1833 {
1834         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1835         struct ccp_dm_workarea exp, src, dst;
1836         struct ccp_op op;
1837         unsigned int sb_count, i_len, o_len;
1838         int ret;
1839
1840         /* Check against the maximum allowable size, in bits */
1841         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1842                 return -EINVAL;
1843
1844         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1845                 return -EINVAL;
1846
1847         memset(&op, 0, sizeof(op));
1848         op.cmd_q = cmd_q;
1849         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1850
1851         /* The RSA modulus must precede the message being acted upon, so
1852          * it must be copied to a DMA area where the message and the
1853          * modulus can be concatenated.  Therefore the input buffer
1854          * length required is twice the output buffer length (which
1855          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1856          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1857          * required.
1858          */
1859         o_len = 32 * ((rsa->key_size + 255) / 256);
1860         i_len = o_len * 2;
1861
1862         sb_count = 0;
1863         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1864                 /* sb_count is the number of storage block slots required
1865                  * for the modulus.
1866                  */
1867                 sb_count = o_len / CCP_SB_BYTES;
1868                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1869                                                                 sb_count);
1870                 if (!op.sb_key)
1871                         return -EIO;
1872         } else {
1873                 /* A version 5 device allows a modulus size that will not fit
1874                  * in the LSB, so the command will transfer it from memory.
1875                  * Set the sb key to the default, even though it's not used.
1876                  */
1877                 op.sb_key = cmd_q->sb_key;
1878         }
1879
1880         /* The RSA exponent must be in little endian format. Reverse its
1881          * byte order.
1882          */
1883         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1884         if (ret)
1885                 goto e_sb;
1886
1887         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1888         if (ret)
1889                 goto e_exp;
1890
1891         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1892                 /* Copy the exponent to the local storage block, using
1893                  * as many 32-byte blocks as were allocated above. It's
1894                  * already little endian, so no further change is required.
1895                  */
1896                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1897                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1898                 if (ret) {
1899                         cmd->engine_error = cmd_q->cmd_error;
1900                         goto e_exp;
1901                 }
1902         } else {
1903                 /* The exponent can be retrieved from memory via DMA. */
1904                 op.exp.u.dma.address = exp.dma.address;
1905                 op.exp.u.dma.offset = 0;
1906         }
1907
1908         /* Concatenate the modulus and the message. Both the modulus and
1909          * the operands must be in little endian format.  Since the input
1910          * is in big endian format it must be converted.
1911          */
1912         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1913         if (ret)
1914                 goto e_exp;
1915
1916         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1917         if (ret)
1918                 goto e_src;
1919         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1920         if (ret)
1921                 goto e_src;
1922
1923         /* Prepare the output area for the operation */
1924         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1925         if (ret)
1926                 goto e_src;
1927
1928         op.soc = 1;
1929         op.src.u.dma.address = src.dma.address;
1930         op.src.u.dma.offset = 0;
1931         op.src.u.dma.length = i_len;
1932         op.dst.u.dma.address = dst.dma.address;
1933         op.dst.u.dma.offset = 0;
1934         op.dst.u.dma.length = o_len;
1935
1936         op.u.rsa.mod_size = rsa->key_size;
1937         op.u.rsa.input_len = i_len;
1938
1939         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1940         if (ret) {
1941                 cmd->engine_error = cmd_q->cmd_error;
1942                 goto e_dst;
1943         }
1944
1945         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1946
1947 e_dst:
1948         ccp_dm_free(&dst);
1949
1950 e_src:
1951         ccp_dm_free(&src);
1952
1953 e_exp:
1954         ccp_dm_free(&exp);
1955
1956 e_sb:
1957         if (sb_count)
1958                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1959
1960         return ret;
1961 }
1962
1963 static noinline_for_stack int
1964 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1965 {
1966         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1967         struct ccp_dm_workarea mask;
1968         struct ccp_data src, dst;
1969         struct ccp_op op;
1970         bool in_place = false;
1971         unsigned int i;
1972         int ret = 0;
1973
1974         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1975                 return -EINVAL;
1976
1977         if (!pt->src || !pt->dst)
1978                 return -EINVAL;
1979
1980         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1982                         return -EINVAL;
1983                 if (!pt->mask)
1984                         return -EINVAL;
1985         }
1986
1987         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1988
1989         memset(&op, 0, sizeof(op));
1990         op.cmd_q = cmd_q;
1991         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1992
1993         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1994                 /* Load the mask */
1995                 op.sb_key = cmd_q->sb_key;
1996
1997                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1998                                            CCP_PASSTHRU_SB_COUNT *
1999                                            CCP_SB_BYTES,
2000                                            DMA_TO_DEVICE);
2001                 if (ret)
2002                         return ret;
2003
2004                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2005                 if (ret)
2006                         goto e_mask;
2007                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2008                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2009                 if (ret) {
2010                         cmd->engine_error = cmd_q->cmd_error;
2011                         goto e_mask;
2012                 }
2013         }
2014
2015         /* Prepare the input and output data workareas. For in-place
2016          * operations we need to set the dma direction to BIDIRECTIONAL
2017          * and copy the src workarea to the dst workarea.
2018          */
2019         if (sg_virt(pt->src) == sg_virt(pt->dst))
2020                 in_place = true;
2021
2022         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2023                             CCP_PASSTHRU_MASKSIZE,
2024                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2025         if (ret)
2026                 goto e_mask;
2027
2028         if (in_place) {
2029                 dst = src;
2030         } else {
2031                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2032                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2033                 if (ret)
2034                         goto e_src;
2035         }
2036
2037         /* Send data to the CCP Passthru engine
2038          *   Because the CCP engine works on a single source and destination
2039          *   dma address at a time, each entry in the source scatterlist
2040          *   (after the dma_map_sg call) must be less than or equal to the
2041          *   (remaining) length in the destination scatterlist entry and the
2042          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2043          */
2044         dst.sg_wa.sg_used = 0;
2045         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2046                 if (!dst.sg_wa.sg ||
2047                     (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2048                         ret = -EINVAL;
2049                         goto e_dst;
2050                 }
2051
2052                 if (i == src.sg_wa.dma_count) {
2053                         op.eom = 1;
2054                         op.soc = 1;
2055                 }
2056
2057                 op.src.type = CCP_MEMTYPE_SYSTEM;
2058                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2059                 op.src.u.dma.offset = 0;
2060                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2061
2062                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2063                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2064                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2065                 op.dst.u.dma.length = op.src.u.dma.length;
2066
2067                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2068                 if (ret) {
2069                         cmd->engine_error = cmd_q->cmd_error;
2070                         goto e_dst;
2071                 }
2072
2073                 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2074                 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2075                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2076                         dst.sg_wa.sg_used = 0;
2077                 }
2078                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2079         }
2080
2081 e_dst:
2082         if (!in_place)
2083                 ccp_free_data(&dst, cmd_q);
2084
2085 e_src:
2086         ccp_free_data(&src, cmd_q);
2087
2088 e_mask:
2089         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2090                 ccp_dm_free(&mask);
2091
2092         return ret;
2093 }
2094
2095 static noinline_for_stack int
2096 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2097                                       struct ccp_cmd *cmd)
2098 {
2099         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2100         struct ccp_dm_workarea mask;
2101         struct ccp_op op;
2102         int ret;
2103
2104         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2105                 return -EINVAL;
2106
2107         if (!pt->src_dma || !pt->dst_dma)
2108                 return -EINVAL;
2109
2110         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2111                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2112                         return -EINVAL;
2113                 if (!pt->mask)
2114                         return -EINVAL;
2115         }
2116
2117         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2118
2119         memset(&op, 0, sizeof(op));
2120         op.cmd_q = cmd_q;
2121         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2122
2123         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2124                 /* Load the mask */
2125                 op.sb_key = cmd_q->sb_key;
2126
2127                 mask.length = pt->mask_len;
2128                 mask.dma.address = pt->mask;
2129                 mask.dma.length = pt->mask_len;
2130
2131                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2132                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2133                 if (ret) {
2134                         cmd->engine_error = cmd_q->cmd_error;
2135                         return ret;
2136                 }
2137         }
2138
2139         /* Send data to the CCP Passthru engine */
2140         op.eom = 1;
2141         op.soc = 1;
2142
2143         op.src.type = CCP_MEMTYPE_SYSTEM;
2144         op.src.u.dma.address = pt->src_dma;
2145         op.src.u.dma.offset = 0;
2146         op.src.u.dma.length = pt->src_len;
2147
2148         op.dst.type = CCP_MEMTYPE_SYSTEM;
2149         op.dst.u.dma.address = pt->dst_dma;
2150         op.dst.u.dma.offset = 0;
2151         op.dst.u.dma.length = pt->src_len;
2152
2153         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2154         if (ret)
2155                 cmd->engine_error = cmd_q->cmd_error;
2156
2157         return ret;
2158 }
2159
2160 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2161 {
2162         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2163         struct ccp_dm_workarea src, dst;
2164         struct ccp_op op;
2165         int ret;
2166         u8 *save;
2167
2168         if (!ecc->u.mm.operand_1 ||
2169             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2170                 return -EINVAL;
2171
2172         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2173                 if (!ecc->u.mm.operand_2 ||
2174                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2175                         return -EINVAL;
2176
2177         if (!ecc->u.mm.result ||
2178             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2179                 return -EINVAL;
2180
2181         memset(&op, 0, sizeof(op));
2182         op.cmd_q = cmd_q;
2183         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2184
2185         /* Concatenate the modulus and the operands. Both the modulus and
2186          * the operands must be in little endian format.  Since the input
2187          * is in big endian format it must be converted and placed in a
2188          * fixed length buffer.
2189          */
2190         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2191                                    DMA_TO_DEVICE);
2192         if (ret)
2193                 return ret;
2194
2195         /* Save the workarea address since it is updated in order to perform
2196          * the concatenation
2197          */
2198         save = src.address;
2199
2200         /* Copy the ECC modulus */
2201         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2202         if (ret)
2203                 goto e_src;
2204         src.address += CCP_ECC_OPERAND_SIZE;
2205
2206         /* Copy the first operand */
2207         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2208                                       ecc->u.mm.operand_1_len);
2209         if (ret)
2210                 goto e_src;
2211         src.address += CCP_ECC_OPERAND_SIZE;
2212
2213         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2214                 /* Copy the second operand */
2215                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2216                                               ecc->u.mm.operand_2_len);
2217                 if (ret)
2218                         goto e_src;
2219                 src.address += CCP_ECC_OPERAND_SIZE;
2220         }
2221
2222         /* Restore the workarea address */
2223         src.address = save;
2224
2225         /* Prepare the output area for the operation */
2226         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2227                                    DMA_FROM_DEVICE);
2228         if (ret)
2229                 goto e_src;
2230
2231         op.soc = 1;
2232         op.src.u.dma.address = src.dma.address;
2233         op.src.u.dma.offset = 0;
2234         op.src.u.dma.length = src.length;
2235         op.dst.u.dma.address = dst.dma.address;
2236         op.dst.u.dma.offset = 0;
2237         op.dst.u.dma.length = dst.length;
2238
2239         op.u.ecc.function = cmd->u.ecc.function;
2240
2241         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2242         if (ret) {
2243                 cmd->engine_error = cmd_q->cmd_error;
2244                 goto e_dst;
2245         }
2246
2247         ecc->ecc_result = le16_to_cpup(
2248                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2249         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2250                 ret = -EIO;
2251                 goto e_dst;
2252         }
2253
2254         /* Save the ECC result */
2255         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2256                                 CCP_ECC_MODULUS_BYTES);
2257
2258 e_dst:
2259         ccp_dm_free(&dst);
2260
2261 e_src:
2262         ccp_dm_free(&src);
2263
2264         return ret;
2265 }
2266
2267 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2268 {
2269         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2270         struct ccp_dm_workarea src, dst;
2271         struct ccp_op op;
2272         int ret;
2273         u8 *save;
2274
2275         if (!ecc->u.pm.point_1.x ||
2276             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2277             !ecc->u.pm.point_1.y ||
2278             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2279                 return -EINVAL;
2280
2281         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2282                 if (!ecc->u.pm.point_2.x ||
2283                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2284                     !ecc->u.pm.point_2.y ||
2285                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2286                         return -EINVAL;
2287         } else {
2288                 if (!ecc->u.pm.domain_a ||
2289                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2290                         return -EINVAL;
2291
2292                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2293                         if (!ecc->u.pm.scalar ||
2294                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2295                                 return -EINVAL;
2296         }
2297
2298         if (!ecc->u.pm.result.x ||
2299             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2300             !ecc->u.pm.result.y ||
2301             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2302                 return -EINVAL;
2303
2304         memset(&op, 0, sizeof(op));
2305         op.cmd_q = cmd_q;
2306         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2307
2308         /* Concatenate the modulus and the operands. Both the modulus and
2309          * the operands must be in little endian format.  Since the input
2310          * is in big endian format it must be converted and placed in a
2311          * fixed length buffer.
2312          */
2313         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2314                                    DMA_TO_DEVICE);
2315         if (ret)
2316                 return ret;
2317
2318         /* Save the workarea address since it is updated in order to perform
2319          * the concatenation
2320          */
2321         save = src.address;
2322
2323         /* Copy the ECC modulus */
2324         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2325         if (ret)
2326                 goto e_src;
2327         src.address += CCP_ECC_OPERAND_SIZE;
2328
2329         /* Copy the first point X and Y coordinate */
2330         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2331                                       ecc->u.pm.point_1.x_len);
2332         if (ret)
2333                 goto e_src;
2334         src.address += CCP_ECC_OPERAND_SIZE;
2335         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2336                                       ecc->u.pm.point_1.y_len);
2337         if (ret)
2338                 goto e_src;
2339         src.address += CCP_ECC_OPERAND_SIZE;
2340
2341         /* Set the first point Z coordinate to 1 */
2342         *src.address = 0x01;
2343         src.address += CCP_ECC_OPERAND_SIZE;
2344
2345         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2346                 /* Copy the second point X and Y coordinate */
2347                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2348                                               ecc->u.pm.point_2.x_len);
2349                 if (ret)
2350                         goto e_src;
2351                 src.address += CCP_ECC_OPERAND_SIZE;
2352                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2353                                               ecc->u.pm.point_2.y_len);
2354                 if (ret)
2355                         goto e_src;
2356                 src.address += CCP_ECC_OPERAND_SIZE;
2357
2358                 /* Set the second point Z coordinate to 1 */
2359                 *src.address = 0x01;
2360                 src.address += CCP_ECC_OPERAND_SIZE;
2361         } else {
2362                 /* Copy the Domain "a" parameter */
2363                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2364                                               ecc->u.pm.domain_a_len);
2365                 if (ret)
2366                         goto e_src;
2367                 src.address += CCP_ECC_OPERAND_SIZE;
2368
2369                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2370                         /* Copy the scalar value */
2371                         ret = ccp_reverse_set_dm_area(&src, 0,
2372                                                       ecc->u.pm.scalar, 0,
2373                                                       ecc->u.pm.scalar_len);
2374                         if (ret)
2375                                 goto e_src;
2376                         src.address += CCP_ECC_OPERAND_SIZE;
2377                 }
2378         }
2379
2380         /* Restore the workarea address */
2381         src.address = save;
2382
2383         /* Prepare the output area for the operation */
2384         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2385                                    DMA_FROM_DEVICE);
2386         if (ret)
2387                 goto e_src;
2388
2389         op.soc = 1;
2390         op.src.u.dma.address = src.dma.address;
2391         op.src.u.dma.offset = 0;
2392         op.src.u.dma.length = src.length;
2393         op.dst.u.dma.address = dst.dma.address;
2394         op.dst.u.dma.offset = 0;
2395         op.dst.u.dma.length = dst.length;
2396
2397         op.u.ecc.function = cmd->u.ecc.function;
2398
2399         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2400         if (ret) {
2401                 cmd->engine_error = cmd_q->cmd_error;
2402                 goto e_dst;
2403         }
2404
2405         ecc->ecc_result = le16_to_cpup(
2406                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2407         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2408                 ret = -EIO;
2409                 goto e_dst;
2410         }
2411
2412         /* Save the workarea address since it is updated as we walk through
2413          * to copy the point math result
2414          */
2415         save = dst.address;
2416
2417         /* Save the ECC result X and Y coordinates */
2418         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2419                                 CCP_ECC_MODULUS_BYTES);
2420         dst.address += CCP_ECC_OUTPUT_SIZE;
2421         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2422                                 CCP_ECC_MODULUS_BYTES);
2423
2424         /* Restore the workarea address */
2425         dst.address = save;
2426
2427 e_dst:
2428         ccp_dm_free(&dst);
2429
2430 e_src:
2431         ccp_dm_free(&src);
2432
2433         return ret;
2434 }
2435
2436 static noinline_for_stack int
2437 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2438 {
2439         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2440
2441         ecc->ecc_result = 0;
2442
2443         if (!ecc->mod ||
2444             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2445                 return -EINVAL;
2446
2447         switch (ecc->function) {
2448         case CCP_ECC_FUNCTION_MMUL_384BIT:
2449         case CCP_ECC_FUNCTION_MADD_384BIT:
2450         case CCP_ECC_FUNCTION_MINV_384BIT:
2451                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2452
2453         case CCP_ECC_FUNCTION_PADD_384BIT:
2454         case CCP_ECC_FUNCTION_PMUL_384BIT:
2455         case CCP_ECC_FUNCTION_PDBL_384BIT:
2456                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2457
2458         default:
2459                 return -EINVAL;
2460         }
2461 }
2462
2463 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2464 {
2465         int ret;
2466
2467         cmd->engine_error = 0;
2468         cmd_q->cmd_error = 0;
2469         cmd_q->int_rcvd = 0;
2470         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2471
2472         switch (cmd->engine) {
2473         case CCP_ENGINE_AES:
2474                 switch (cmd->u.aes.mode) {
2475                 case CCP_AES_MODE_CMAC:
2476                         ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2477                         break;
2478                 case CCP_AES_MODE_GCM:
2479                         ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2480                         break;
2481                 default:
2482                         ret = ccp_run_aes_cmd(cmd_q, cmd);
2483                         break;
2484                 }
2485                 break;
2486         case CCP_ENGINE_XTS_AES_128:
2487                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2488                 break;
2489         case CCP_ENGINE_DES3:
2490                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2491                 break;
2492         case CCP_ENGINE_SHA:
2493                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2494                 break;
2495         case CCP_ENGINE_RSA:
2496                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2497                 break;
2498         case CCP_ENGINE_PASSTHRU:
2499                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2500                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2501                 else
2502                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2503                 break;
2504         case CCP_ENGINE_ECC:
2505                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2506                 break;
2507         default:
2508                 ret = -EINVAL;
2509         }
2510
2511         return ret;
2512 }