drivers/crypto/cavium/cpt/cptvf_algs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Copyright (C) 2016 Cavium, Inc.
   5  */
   6
   7 #include <crypto/aes.h>
   8 #include <crypto/algapi.h>
   9 #include <crypto/authenc.h>
  10 #include <crypto/internal/des.h>
  11 #include <crypto/xts.h>
  12 #include <linux/crypto.h>
  13 #include <linux/err.h>
  14 #include <linux/list.h>
  15 #include <linux/scatterlist.h>
  16
  17 #include "cptvf.h"
  18 #include "cptvf_algs.h"
  19
  20 struct cpt_device_handle {
  21         void *cdev[MAX_DEVICES];
  22         u32 dev_count;
  23 };
  24
  25 static struct cpt_device_handle dev_handle;
  26
  27 static void cvm_callback(u32 status, void *arg)
  28 {
  29         struct crypto_async_request *req = (struct crypto_async_request *)arg;
  30
  31         crypto_request_complete(req, !status);
  32 }
  33
  34 static inline void update_input_iv(struct cpt_request_info *req_info,
  35                                    u8 *iv, u32 enc_iv_len,
  36                                    u32 *argcnt)
  37 {
  38         /* Setting the iv information */
  39         req_info->in[*argcnt].vptr = (void *)iv;
  40         req_info->in[*argcnt].size = enc_iv_len;
  41         req_info->req.dlen += enc_iv_len;
  42
  43         ++(*argcnt);
  44 }
  45
  46 static inline void update_output_iv(struct cpt_request_info *req_info,
  47                                     u8 *iv, u32 enc_iv_len,
  48                                     u32 *argcnt)
  49 {
  50         /* Setting the iv information */
  51         req_info->out[*argcnt].vptr = (void *)iv;
  52         req_info->out[*argcnt].size = enc_iv_len;
  53         req_info->rlen += enc_iv_len;
  54
  55         ++(*argcnt);
  56 }
  57
  58 static inline void update_input_data(struct cpt_request_info *req_info,
  59                                      struct scatterlist *inp_sg,
  60                                      u32 nbytes, u32 *argcnt)
  61 {
  62         req_info->req.dlen += nbytes;
  63
  64         while (nbytes) {
  65                 u32 len = min(nbytes, inp_sg->length);
  66                 u8 *ptr = sg_virt(inp_sg);
  67
  68                 req_info->in[*argcnt].vptr = (void *)ptr;
  69                 req_info->in[*argcnt].size = len;
  70                 nbytes -= len;
  71
  72                 ++(*argcnt);
  73                 ++inp_sg;
  74         }
  75 }
  76
  77 static inline void update_output_data(struct cpt_request_info *req_info,
  78                                       struct scatterlist *outp_sg,
  79                                       u32 nbytes, u32 *argcnt)
  80 {
  81         req_info->rlen += nbytes;
  82
  83         while (nbytes) {
  84                 u32 len = min(nbytes, outp_sg->length);
  85                 u8 *ptr = sg_virt(outp_sg);
  86
  87                 req_info->out[*argcnt].vptr = (void *)ptr;
  88                 req_info->out[*argcnt].size = len;
  89                 nbytes -= len;
  90                 ++(*argcnt);
  91                 ++outp_sg;
  92         }
  93 }
  94
  95 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
  96                                  u32 *argcnt)
  97 {
  98         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  99         struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 100         struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
 101         struct fc_context *fctx = &rctx->fctx;
 102         u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
 103         struct cpt_request_info *req_info = &rctx->cpt_req;
 104         __be64 *ctrl_flags = NULL;
 105         __be64 *offset_control;
 106
 107         req_info->ctrl.s.grp = 0;
 108         req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
 109         req_info->ctrl.s.se_req = SE_CORE_REQ;
 110
 111         req_info->req.opcode.s.major = MAJOR_OP_FC |
 112                                         DMA_MODE_FLAG(DMA_GATHER_SCATTER);
 113         if (enc)
 114                 req_info->req.opcode.s.minor = 2;
 115         else
 116                 req_info->req.opcode.s.minor = 3;
 117
 118         req_info->req.param1 = req->cryptlen; /* Encryption Data length */
 119         req_info->req.param2 = 0; /*Auth data length */
 120
 121         fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
 122         fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
 123         fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
 124
 125         if (ctx->cipher_type == AES_XTS)
 126                 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
 127         else
 128                 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
 129         ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags;
 130         *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags);
 131
 132         offset_control = (__be64 *)&rctx->control_word;
 133         *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
 134         /* Storing  Packet Data Information in offset
 135          * Control Word First 8 bytes
 136          */
 137         req_info->in[*argcnt].vptr = (u8 *)offset_control;
 138         req_info->in[*argcnt].size = CONTROL_WORD_LEN;
 139         req_info->req.dlen += CONTROL_WORD_LEN;
 140         ++(*argcnt);
 141
 142         req_info->in[*argcnt].vptr = (u8 *)fctx;
 143         req_info->in[*argcnt].size = sizeof(struct fc_context);
 144         req_info->req.dlen += sizeof(struct fc_context);
 145
 146         ++(*argcnt);
 147
 148         return 0;
 149 }
 150
 151 static inline u32 create_input_list(struct skcipher_request  *req, u32 enc,
 152                                     u32 enc_iv_len)
 153 {
 154         struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
 155         struct cpt_request_info *req_info = &rctx->cpt_req;
 156         u32 argcnt =  0;
 157
 158         create_ctx_hdr(req, enc, &argcnt);
 159         update_input_iv(req_info, req->iv, enc_iv_len, &argcnt);
 160         update_input_data(req_info, req->src, req->cryptlen, &argcnt);
 161         req_info->incnt = argcnt;
 162
 163         return 0;
 164 }
 165
 166 static inline void store_cb_info(struct skcipher_request *req,
 167                                  struct cpt_request_info *req_info)
 168 {
 169         req_info->callback = (void *)cvm_callback;
 170         req_info->callback_arg = (void *)&req->base;
 171 }
 172
 173 static inline void create_output_list(struct skcipher_request *req,
 174                                       u32 enc_iv_len)
 175 {
 176         struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
 177         struct cpt_request_info *req_info = &rctx->cpt_req;
 178         u32 argcnt = 0;
 179
 180         /* OUTPUT Buffer Processing
 181          * AES encryption/decryption output would be
 182          * received in the following format
 183          *
 184          * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
 185          * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
 186          */
 187         /* Reading IV information */
 188         update_output_iv(req_info, req->iv, enc_iv_len, &argcnt);
 189         update_output_data(req_info, req->dst, req->cryptlen, &argcnt);
 190         req_info->outcnt = argcnt;
 191 }
 192
 193 static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc)
 194 {
 195         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 196         struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
 197         u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
 198         struct fc_context *fctx = &rctx->fctx;
 199         struct cpt_request_info *req_info = &rctx->cpt_req;
 200         void *cdev = NULL;
 201         int status;
 202
 203         memset(req_info, 0, sizeof(struct cpt_request_info));
 204         req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0;
 205         memset(fctx, 0, sizeof(struct fc_context));
 206         create_input_list(req, enc, enc_iv_len);
 207         create_output_list(req, enc_iv_len);
 208         store_cb_info(req, req_info);
 209         cdev = dev_handle.cdev[smp_processor_id()];
 210         status = cptvf_do_request(cdev, req_info);
 211         /* We perform an asynchronous send and once
 212          * the request is completed the driver would
 213          * intimate through  registered call back functions
 214          */
 215
 216         if (status)
 217                 return status;
 218         else
 219                 return -EINPROGRESS;
 220 }
 221
 222 static int cvm_encrypt(struct skcipher_request *req)
 223 {
 224         return cvm_enc_dec(req, true);
 225 }
 226
 227 static int cvm_decrypt(struct skcipher_request *req)
 228 {
 229         return cvm_enc_dec(req, false);
 230 }
 231
 232 static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
 233                    u32 keylen)
 234 {
 235         struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
 236         int err;
 237         const u8 *key1 = key;
 238         const u8 *key2 = key + (keylen / 2);
 239
 240         err = xts_verify_key(cipher, key, keylen);
 241         if (err)
 242                 return err;
 243         ctx->key_len = keylen;
 244         memcpy(ctx->enc_key, key1, keylen / 2);
 245         memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
 246         ctx->cipher_type = AES_XTS;
 247         switch (ctx->key_len) {
 248         case 32:
 249                 ctx->key_type = AES_128_BIT;
 250                 break;
 251         case 64:
 252                 ctx->key_type = AES_256_BIT;
 253                 break;
 254         default:
 255                 return -EINVAL;
 256         }
 257
 258         return 0;
 259 }
 260
 261 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
 262 {
 263         if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
 264                 ctx->key_len = keylen;
 265                 switch (ctx->key_len) {
 266                 case 16:
 267                         ctx->key_type = AES_128_BIT;
 268                         break;
 269                 case 24:
 270                         ctx->key_type = AES_192_BIT;
 271                         break;
 272                 case 32:
 273                         ctx->key_type = AES_256_BIT;
 274                         break;
 275                 default:
 276                         return -EINVAL;
 277                 }
 278
 279                 if (ctx->cipher_type == DES3_CBC)
 280                         ctx->key_type = 0;
 281
 282                 return 0;
 283         }
 284
 285         return -EINVAL;
 286 }
 287
 288 static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key,
 289                       u32 keylen, u8 cipher_type)
 290 {
 291         struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
 292
 293         ctx->cipher_type = cipher_type;
 294         if (!cvm_validate_keylen(ctx, keylen)) {
 295                 memcpy(ctx->enc_key, key, keylen);
 296                 return 0;
 297         } else {
 298                 return -EINVAL;
 299         }
 300 }
 301
 302 static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
 303                               u32 keylen)
 304 {
 305         return cvm_setkey(cipher, key, keylen, AES_CBC);
 306 }
 307
 308 static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
 309                               u32 keylen)
 310 {
 311         return cvm_setkey(cipher, key, keylen, AES_ECB);
 312 }
 313
 314 static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
 315                                u32 keylen)
 316 {
 317         return verify_skcipher_des3_key(cipher, key) ?:
 318                cvm_setkey(cipher, key, keylen, DES3_CBC);
 319 }
 320
 321 static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
 322                                u32 keylen)
 323 {
 324         return verify_skcipher_des3_key(cipher, key) ?:
 325                cvm_setkey(cipher, key, keylen, DES3_ECB);
 326 }
 327
 328 static int cvm_enc_dec_init(struct crypto_skcipher *tfm)
 329 {
 330         crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx));
 331
 332         return 0;
 333 }
 334
 335 static struct skcipher_alg algs[] = { {
 336         .base.cra_flags         = CRYPTO_ALG_ASYNC |
 337                                   CRYPTO_ALG_ALLOCATES_MEMORY,
 338         .base.cra_blocksize     = AES_BLOCK_SIZE,
 339         .base.cra_ctxsize       = sizeof(struct cvm_enc_ctx),
 340         .base.cra_alignmask     = 7,
 341         .base.cra_priority      = 4001,
 342         .base.cra_name          = "xts(aes)",
 343         .base.cra_driver_name   = "cavium-xts-aes",
 344         .base.cra_module        = THIS_MODULE,
 345
 346         .ivsize                 = AES_BLOCK_SIZE,
 347         .min_keysize            = 2 * AES_MIN_KEY_SIZE,
 348         .max_keysize            = 2 * AES_MAX_KEY_SIZE,
 349         .setkey                 = cvm_xts_setkey,
 350         .encrypt                = cvm_encrypt,
 351         .decrypt                = cvm_decrypt,
 352         .init                   = cvm_enc_dec_init,
 353 }, {
 354         .base.cra_flags         = CRYPTO_ALG_ASYNC |
 355                                   CRYPTO_ALG_ALLOCATES_MEMORY,
 356         .base.cra_blocksize     = AES_BLOCK_SIZE,
 357         .base.cra_ctxsize       = sizeof(struct cvm_enc_ctx),
 358         .base.cra_alignmask     = 7,
 359         .base.cra_priority      = 4001,
 360         .base.cra_name          = "cbc(aes)",
 361         .base.cra_driver_name   = "cavium-cbc-aes",
 362         .base.cra_module        = THIS_MODULE,
 363
 364         .ivsize                 = AES_BLOCK_SIZE,
 365         .min_keysize            = AES_MIN_KEY_SIZE,
 366         .max_keysize            = AES_MAX_KEY_SIZE,
 367         .setkey                 = cvm_cbc_aes_setkey,
 368         .encrypt                = cvm_encrypt,
 369         .decrypt                = cvm_decrypt,
 370         .init                   = cvm_enc_dec_init,
 371 }, {
 372         .base.cra_flags         = CRYPTO_ALG_ASYNC |
 373                                   CRYPTO_ALG_ALLOCATES_MEMORY,
 374         .base.cra_blocksize     = AES_BLOCK_SIZE,
 375         .base.cra_ctxsize       = sizeof(struct cvm_enc_ctx),
 376         .base.cra_alignmask     = 7,
 377         .base.cra_priority      = 4001,
 378         .base.cra_name          = "ecb(aes)",
 379         .base.cra_driver_name   = "cavium-ecb-aes",
 380         .base.cra_module        = THIS_MODULE,
 381
 382         .min_keysize            = AES_MIN_KEY_SIZE,
 383         .max_keysize            = AES_MAX_KEY_SIZE,
 384         .setkey                 = cvm_ecb_aes_setkey,
 385         .encrypt                = cvm_encrypt,
 386         .decrypt                = cvm_decrypt,
 387         .init                   = cvm_enc_dec_init,
 388 }, {
 389         .base.cra_flags         = CRYPTO_ALG_ASYNC |
 390                                   CRYPTO_ALG_ALLOCATES_MEMORY,
 391         .base.cra_blocksize     = DES3_EDE_BLOCK_SIZE,
 392         .base.cra_ctxsize       = sizeof(struct cvm_des3_ctx),
 393         .base.cra_alignmask     = 7,
 394         .base.cra_priority      = 4001,
 395         .base.cra_name          = "cbc(des3_ede)",
 396         .base.cra_driver_name   = "cavium-cbc-des3_ede",
 397         .base.cra_module        = THIS_MODULE,
 398
 399         .min_keysize            = DES3_EDE_KEY_SIZE,
 400         .max_keysize            = DES3_EDE_KEY_SIZE,
 401         .ivsize                 = DES_BLOCK_SIZE,
 402         .setkey                 = cvm_cbc_des3_setkey,
 403         .encrypt                = cvm_encrypt,
 404         .decrypt                = cvm_decrypt,
 405         .init                   = cvm_enc_dec_init,
 406 }, {
 407         .base.cra_flags         = CRYPTO_ALG_ASYNC |
 408                                   CRYPTO_ALG_ALLOCATES_MEMORY,
 409         .base.cra_blocksize     = DES3_EDE_BLOCK_SIZE,
 410         .base.cra_ctxsize       = sizeof(struct cvm_des3_ctx),
 411         .base.cra_alignmask     = 7,
 412         .base.cra_priority      = 4001,
 413         .base.cra_name          = "ecb(des3_ede)",
 414         .base.cra_driver_name   = "cavium-ecb-des3_ede",
 415         .base.cra_module        = THIS_MODULE,
 416
 417         .min_keysize            = DES3_EDE_KEY_SIZE,
 418         .max_keysize            = DES3_EDE_KEY_SIZE,
 419         .ivsize                 = DES_BLOCK_SIZE,
 420         .setkey                 = cvm_ecb_des3_setkey,
 421         .encrypt                = cvm_encrypt,
 422         .decrypt                = cvm_decrypt,
 423         .init                   = cvm_enc_dec_init,
 424 } };
 425
 426 static inline int cav_register_algs(void)
 427 {
 428         return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
 429 }
 430
 431 static inline void cav_unregister_algs(void)
 432 {
 433         crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 434 }
 435
 436 int cvm_crypto_init(struct cpt_vf *cptvf)
 437 {
 438         struct pci_dev *pdev = cptvf->pdev;
 439         u32 dev_count;
 440
 441         dev_count = dev_handle.dev_count;
 442         dev_handle.cdev[dev_count] = cptvf;
 443         dev_handle.dev_count++;
 444
 445         if (dev_count == 3) {
 446                 if (cav_register_algs()) {
 447                         dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
 448                         return -EINVAL;
 449                 }
 450         }
 451
 452         return 0;
 453 }
 454
 455 void cvm_crypto_exit(void)
 456 {
 457         u32 dev_count;
 458
 459         dev_count = --dev_handle.dev_count;
 460         if (!dev_count)
 461                 cav_unregister_algs();
 462 }