drivers/crypto/stm32/stm32-cryp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) STMicroelectronics SA 2017
   4  * Author: Fabien Dessenne <fabien.dessenne@st.com>
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/delay.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/iopoll.h>
  11 #include <linux/module.h>
  12 #include <linux/of_device.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/pm_runtime.h>
  15 #include <linux/reset.h>
  16
  17 #include <crypto/aes.h>
  18 #include <crypto/internal/des.h>
  19 #include <crypto/engine.h>
  20 #include <crypto/scatterwalk.h>
  21 #include <crypto/internal/aead.h>
  22 #include <crypto/internal/skcipher.h>
  23
  24 #define DRIVER_NAME             "stm32-cryp"
  25
  26 /* Bit [0] encrypt / decrypt */
  27 #define FLG_ENCRYPT             BIT(0)
  28 /* Bit [8..1] algo & operation mode */
  29 #define FLG_AES                 BIT(1)
  30 #define FLG_DES                 BIT(2)
  31 #define FLG_TDES                BIT(3)
  32 #define FLG_ECB                 BIT(4)
  33 #define FLG_CBC                 BIT(5)
  34 #define FLG_CTR                 BIT(6)
  35 #define FLG_GCM                 BIT(7)
  36 #define FLG_CCM                 BIT(8)
  37 /* Mode mask = bits [15..0] */
  38 #define FLG_MODE_MASK           GENMASK(15, 0)
  39 /* Bit [31..16] status  */
  40
  41 /* Registers */
  42 #define CRYP_CR                 0x00000000
  43 #define CRYP_SR                 0x00000004
  44 #define CRYP_DIN                0x00000008
  45 #define CRYP_DOUT               0x0000000C
  46 #define CRYP_DMACR              0x00000010
  47 #define CRYP_IMSCR              0x00000014
  48 #define CRYP_RISR               0x00000018
  49 #define CRYP_MISR               0x0000001C
  50 #define CRYP_K0LR               0x00000020
  51 #define CRYP_K0RR               0x00000024
  52 #define CRYP_K1LR               0x00000028
  53 #define CRYP_K1RR               0x0000002C
  54 #define CRYP_K2LR               0x00000030
  55 #define CRYP_K2RR               0x00000034
  56 #define CRYP_K3LR               0x00000038
  57 #define CRYP_K3RR               0x0000003C
  58 #define CRYP_IV0LR              0x00000040
  59 #define CRYP_IV0RR              0x00000044
  60 #define CRYP_IV1LR              0x00000048
  61 #define CRYP_IV1RR              0x0000004C
  62 #define CRYP_CSGCMCCM0R         0x00000050
  63 #define CRYP_CSGCM0R            0x00000070
  64
  65 /* Registers values */
  66 #define CR_DEC_NOT_ENC          0x00000004
  67 #define CR_TDES_ECB             0x00000000
  68 #define CR_TDES_CBC             0x00000008
  69 #define CR_DES_ECB              0x00000010
  70 #define CR_DES_CBC              0x00000018
  71 #define CR_AES_ECB              0x00000020
  72 #define CR_AES_CBC              0x00000028
  73 #define CR_AES_CTR              0x00000030
  74 #define CR_AES_KP               0x00000038
  75 #define CR_AES_GCM              0x00080000
  76 #define CR_AES_CCM              0x00080008
  77 #define CR_AES_UNKNOWN          0xFFFFFFFF
  78 #define CR_ALGO_MASK            0x00080038
  79 #define CR_DATA32               0x00000000
  80 #define CR_DATA16               0x00000040
  81 #define CR_DATA8                0x00000080
  82 #define CR_DATA1                0x000000C0
  83 #define CR_KEY128               0x00000000
  84 #define CR_KEY192               0x00000100
  85 #define CR_KEY256               0x00000200
  86 #define CR_FFLUSH               0x00004000
  87 #define CR_CRYPEN               0x00008000
  88 #define CR_PH_INIT              0x00000000
  89 #define CR_PH_HEADER            0x00010000
  90 #define CR_PH_PAYLOAD           0x00020000
  91 #define CR_PH_FINAL             0x00030000
  92 #define CR_PH_MASK              0x00030000
  93 #define CR_NBPBL_SHIFT          20
  94
  95 #define SR_BUSY                 0x00000010
  96 #define SR_OFNE                 0x00000004
  97
  98 #define IMSCR_IN                BIT(0)
  99 #define IMSCR_OUT               BIT(1)
 100
 101 #define MISR_IN                 BIT(0)
 102 #define MISR_OUT                BIT(1)
 103
 104 /* Misc */
 105 #define AES_BLOCK_32            (AES_BLOCK_SIZE / sizeof(u32))
 106 #define GCM_CTR_INIT            2
 107 #define CRYP_AUTOSUSPEND_DELAY  50
 108
 109 struct stm32_cryp_caps {
 110         bool                    swap_final;
 111         bool                    padding_wa;
 112 };
 113
 114 struct stm32_cryp_ctx {
 115         struct crypto_engine_ctx enginectx;
 116         struct stm32_cryp       *cryp;
 117         int                     keylen;
 118         __be32                  key[AES_KEYSIZE_256 / sizeof(u32)];
 119         unsigned long           flags;
 120 };
 121
 122 struct stm32_cryp_reqctx {
 123         unsigned long mode;
 124 };
 125
 126 struct stm32_cryp {
 127         struct list_head        list;
 128         struct device           *dev;
 129         void __iomem            *regs;
 130         struct clk              *clk;
 131         unsigned long           flags;
 132         u32                     irq_status;
 133         const struct stm32_cryp_caps *caps;
 134         struct stm32_cryp_ctx   *ctx;
 135
 136         struct crypto_engine    *engine;
 137
 138         struct skcipher_request *req;
 139         struct aead_request     *areq;
 140
 141         size_t                  authsize;
 142         size_t                  hw_blocksize;
 143
 144         size_t                  payload_in;
 145         size_t                  header_in;
 146         size_t                  payload_out;
 147
 148         struct scatterlist      *out_sg;
 149
 150         struct scatter_walk     in_walk;
 151         struct scatter_walk     out_walk;
 152
 153         __be32                  last_ctr[4];
 154         u32                     gcm_ctr;
 155 };
 156
 157 struct stm32_cryp_list {
 158         struct list_head        dev_list;
 159         spinlock_t              lock; /* protect dev_list */
 160 };
 161
 162 static struct stm32_cryp_list cryp_list = {
 163         .dev_list = LIST_HEAD_INIT(cryp_list.dev_list),
 164         .lock     = __SPIN_LOCK_UNLOCKED(cryp_list.lock),
 165 };
 166
 167 static inline bool is_aes(struct stm32_cryp *cryp)
 168 {
 169         return cryp->flags & FLG_AES;
 170 }
 171
 172 static inline bool is_des(struct stm32_cryp *cryp)
 173 {
 174         return cryp->flags & FLG_DES;
 175 }
 176
 177 static inline bool is_tdes(struct stm32_cryp *cryp)
 178 {
 179         return cryp->flags & FLG_TDES;
 180 }
 181
 182 static inline bool is_ecb(struct stm32_cryp *cryp)
 183 {
 184         return cryp->flags & FLG_ECB;
 185 }
 186
 187 static inline bool is_cbc(struct stm32_cryp *cryp)
 188 {
 189         return cryp->flags & FLG_CBC;
 190 }
 191
 192 static inline bool is_ctr(struct stm32_cryp *cryp)
 193 {
 194         return cryp->flags & FLG_CTR;
 195 }
 196
 197 static inline bool is_gcm(struct stm32_cryp *cryp)
 198 {
 199         return cryp->flags & FLG_GCM;
 200 }
 201
 202 static inline bool is_ccm(struct stm32_cryp *cryp)
 203 {
 204         return cryp->flags & FLG_CCM;
 205 }
 206
 207 static inline bool is_encrypt(struct stm32_cryp *cryp)
 208 {
 209         return cryp->flags & FLG_ENCRYPT;
 210 }
 211
 212 static inline bool is_decrypt(struct stm32_cryp *cryp)
 213 {
 214         return !is_encrypt(cryp);
 215 }
 216
 217 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)
 218 {
 219         return readl_relaxed(cryp->regs + ofst);
 220 }
 221
 222 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)
 223 {
 224         writel_relaxed(val, cryp->regs + ofst);
 225 }
 226
 227 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
 228 {
 229         u32 status;
 230
 231         return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
 232                         !(status & SR_BUSY), 10, 100000);
 233 }
 234
 235 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)
 236 {
 237         u32 status;
 238
 239         return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status,
 240                         !(status & CR_CRYPEN), 10, 100000);
 241 }
 242
 243 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
 244 {
 245         u32 status;
 246
 247         return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
 248                         status & SR_OFNE, 10, 100000);
 249 }
 250
 251 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
 252 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
 253
 254 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
 255 {
 256         struct stm32_cryp *tmp, *cryp = NULL;
 257
 258         spin_lock_bh(&cryp_list.lock);
 259         if (!ctx->cryp) {
 260                 list_for_each_entry(tmp, &cryp_list.dev_list, list) {
 261                         cryp = tmp;
 262                         break;
 263                 }
 264                 ctx->cryp = cryp;
 265         } else {
 266                 cryp = ctx->cryp;
 267         }
 268
 269         spin_unlock_bh(&cryp_list.lock);
 270
 271         return cryp;
 272 }
 273
 274 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)
 275 {
 276         if (!iv)
 277                 return;
 278
 279         stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++));
 280         stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++));
 281
 282         if (is_aes(cryp)) {
 283                 stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++));
 284                 stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++));
 285         }
 286 }
 287
 288 static void stm32_cryp_get_iv(struct stm32_cryp *cryp)
 289 {
 290         struct skcipher_request *req = cryp->req;
 291         __be32 *tmp = (void *)req->iv;
 292
 293         if (!tmp)
 294                 return;
 295
 296         *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
 297         *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
 298
 299         if (is_aes(cryp)) {
 300                 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
 301                 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
 302         }
 303 }
 304
 305 static void stm32_cryp_hw_write_key(struct stm32_cryp *c)
 306 {
 307         unsigned int i;
 308         int r_id;
 309
 310         if (is_des(c)) {
 311                 stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0]));
 312                 stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1]));
 313         } else {
 314                 r_id = CRYP_K3RR;
 315                 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
 316                         stm32_cryp_write(c, r_id,
 317                                          be32_to_cpu(c->ctx->key[i - 1]));
 318         }
 319 }
 320
 321 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)
 322 {
 323         if (is_aes(cryp) && is_ecb(cryp))
 324                 return CR_AES_ECB;
 325
 326         if (is_aes(cryp) && is_cbc(cryp))
 327                 return CR_AES_CBC;
 328
 329         if (is_aes(cryp) && is_ctr(cryp))
 330                 return CR_AES_CTR;
 331
 332         if (is_aes(cryp) && is_gcm(cryp))
 333                 return CR_AES_GCM;
 334
 335         if (is_aes(cryp) && is_ccm(cryp))
 336                 return CR_AES_CCM;
 337
 338         if (is_des(cryp) && is_ecb(cryp))
 339                 return CR_DES_ECB;
 340
 341         if (is_des(cryp) && is_cbc(cryp))
 342                 return CR_DES_CBC;
 343
 344         if (is_tdes(cryp) && is_ecb(cryp))
 345                 return CR_TDES_ECB;
 346
 347         if (is_tdes(cryp) && is_cbc(cryp))
 348                 return CR_TDES_CBC;
 349
 350         dev_err(cryp->dev, "Unknown mode\n");
 351         return CR_AES_UNKNOWN;
 352 }
 353
 354 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp)
 355 {
 356         return is_encrypt(cryp) ? cryp->areq->cryptlen :
 357                                   cryp->areq->cryptlen - cryp->authsize;
 358 }
 359
 360 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
 361 {
 362         int ret;
 363         __be32 iv[4];
 364
 365         /* Phase 1 : init */
 366         memcpy(iv, cryp->areq->iv, 12);
 367         iv[3] = cpu_to_be32(GCM_CTR_INIT);
 368         cryp->gcm_ctr = GCM_CTR_INIT;
 369         stm32_cryp_hw_write_iv(cryp, iv);
 370
 371         stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
 372
 373         /* Wait for end of processing */
 374         ret = stm32_cryp_wait_enable(cryp);
 375         if (ret) {
 376                 dev_err(cryp->dev, "Timeout (gcm init)\n");
 377                 return ret;
 378         }
 379
 380         /* Prepare next phase */
 381         if (cryp->areq->assoclen) {
 382                 cfg |= CR_PH_HEADER;
 383                 stm32_cryp_write(cryp, CRYP_CR, cfg);
 384         } else if (stm32_cryp_get_input_text_len(cryp)) {
 385                 cfg |= CR_PH_PAYLOAD;
 386                 stm32_cryp_write(cryp, CRYP_CR, cfg);
 387         }
 388
 389         return 0;
 390 }
 391
 392 static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp)
 393 {
 394         u32 cfg;
 395         int err;
 396
 397         /* Check if whole header written */
 398         if (!cryp->header_in) {
 399                 /* Wait for completion */
 400                 err = stm32_cryp_wait_busy(cryp);
 401                 if (err) {
 402                         dev_err(cryp->dev, "Timeout (gcm/ccm header)\n");
 403                         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
 404                         stm32_cryp_finish_req(cryp, err);
 405                         return;
 406                 }
 407
 408                 if (stm32_cryp_get_input_text_len(cryp)) {
 409                         /* Phase 3 : payload */
 410                         cfg = stm32_cryp_read(cryp, CRYP_CR);
 411                         cfg &= ~CR_CRYPEN;
 412                         stm32_cryp_write(cryp, CRYP_CR, cfg);
 413
 414                         cfg &= ~CR_PH_MASK;
 415                         cfg |= CR_PH_PAYLOAD | CR_CRYPEN;
 416                         stm32_cryp_write(cryp, CRYP_CR, cfg);
 417                 } else {
 418                         /*
 419                          * Phase 4 : tag.
 420                          * Nothing to read, nothing to write, caller have to
 421                          * end request
 422                          */
 423                 }
 424         }
 425 }
 426
 427 static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp)
 428 {
 429         unsigned int i;
 430         size_t written;
 431         size_t len;
 432         u32 alen = cryp->areq->assoclen;
 433         u32 block[AES_BLOCK_32] = {0};
 434         u8 *b8 = (u8 *)block;
 435
 436         if (alen <= 65280) {
 437                 /* Write first u32 of B1 */
 438                 b8[0] = (alen >> 8) & 0xFF;
 439                 b8[1] = alen & 0xFF;
 440                 len = 2;
 441         } else {
 442                 /* Build the two first u32 of B1 */
 443                 b8[0] = 0xFF;
 444                 b8[1] = 0xFE;
 445                 b8[2] = (alen & 0xFF000000) >> 24;
 446                 b8[3] = (alen & 0x00FF0000) >> 16;
 447                 b8[4] = (alen & 0x0000FF00) >> 8;
 448                 b8[5] = alen & 0x000000FF;
 449                 len = 6;
 450         }
 451
 452         written = min_t(size_t, AES_BLOCK_SIZE - len, alen);
 453
 454         scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0);
 455         for (i = 0; i < AES_BLOCK_32; i++)
 456                 stm32_cryp_write(cryp, CRYP_DIN, block[i]);
 457
 458         cryp->header_in -= written;
 459
 460         stm32_crypt_gcmccm_end_header(cryp);
 461 }
 462
 463 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
 464 {
 465         int ret;
 466         u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32];
 467         u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32;
 468         __be32 *bd;
 469         u32 *d;
 470         unsigned int i, textlen;
 471
 472         /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */
 473         memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
 474         memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
 475         iv[AES_BLOCK_SIZE - 1] = 1;
 476         stm32_cryp_hw_write_iv(cryp, (__be32 *)iv);
 477
 478         /* Build B0 */
 479         memcpy(b0, iv, AES_BLOCK_SIZE);
 480
 481         b0[0] |= (8 * ((cryp->authsize - 2) / 2));
 482
 483         if (cryp->areq->assoclen)
 484                 b0[0] |= 0x40;
 485
 486         textlen = stm32_cryp_get_input_text_len(cryp);
 487
 488         b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
 489         b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
 490
 491         /* Enable HW */
 492         stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
 493
 494         /* Write B0 */
 495         d = (u32 *)b0;
 496         bd = (__be32 *)b0;
 497
 498         for (i = 0; i < AES_BLOCK_32; i++) {
 499                 u32 xd = d[i];
 500
 501                 if (!cryp->caps->padding_wa)
 502                         xd = be32_to_cpu(bd[i]);
 503                 stm32_cryp_write(cryp, CRYP_DIN, xd);
 504         }
 505
 506         /* Wait for end of processing */
 507         ret = stm32_cryp_wait_enable(cryp);
 508         if (ret) {
 509                 dev_err(cryp->dev, "Timeout (ccm init)\n");
 510                 return ret;
 511         }
 512
 513         /* Prepare next phase */
 514         if (cryp->areq->assoclen) {
 515                 cfg |= CR_PH_HEADER | CR_CRYPEN;
 516                 stm32_cryp_write(cryp, CRYP_CR, cfg);
 517
 518                 /* Write first (special) block (may move to next phase [payload]) */
 519                 stm32_cryp_write_ccm_first_header(cryp);
 520         } else if (stm32_cryp_get_input_text_len(cryp)) {
 521                 cfg |= CR_PH_PAYLOAD;
 522                 stm32_cryp_write(cryp, CRYP_CR, cfg);
 523         }
 524
 525         return 0;
 526 }
 527
 528 static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
 529 {
 530         int ret;
 531         u32 cfg, hw_mode;
 532
 533         pm_runtime_get_sync(cryp->dev);
 534
 535         /* Disable interrupt */
 536         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
 537
 538         /* Set key */
 539         stm32_cryp_hw_write_key(cryp);
 540
 541         /* Set configuration */
 542         cfg = CR_DATA8 | CR_FFLUSH;
 543
 544         switch (cryp->ctx->keylen) {
 545         case AES_KEYSIZE_128:
 546                 cfg |= CR_KEY128;
 547                 break;
 548
 549         case AES_KEYSIZE_192:
 550                 cfg |= CR_KEY192;
 551                 break;
 552
 553         default:
 554         case AES_KEYSIZE_256:
 555                 cfg |= CR_KEY256;
 556                 break;
 557         }
 558
 559         hw_mode = stm32_cryp_get_hw_mode(cryp);
 560         if (hw_mode == CR_AES_UNKNOWN)
 561                 return -EINVAL;
 562
 563         /* AES ECB/CBC decrypt: run key preparation first */
 564         if (is_decrypt(cryp) &&
 565             ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
 566                 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
 567
 568                 /* Wait for end of processing */
 569                 ret = stm32_cryp_wait_busy(cryp);
 570                 if (ret) {
 571                         dev_err(cryp->dev, "Timeout (key preparation)\n");
 572                         return ret;
 573                 }
 574         }
 575
 576         cfg |= hw_mode;
 577
 578         if (is_decrypt(cryp))
 579                 cfg |= CR_DEC_NOT_ENC;
 580
 581         /* Apply config and flush (valid when CRYPEN = 0) */
 582         stm32_cryp_write(cryp, CRYP_CR, cfg);
 583
 584         switch (hw_mode) {
 585         case CR_AES_GCM:
 586         case CR_AES_CCM:
 587                 /* Phase 1 : init */
 588                 if (hw_mode == CR_AES_CCM)
 589                         ret = stm32_cryp_ccm_init(cryp, cfg);
 590                 else
 591                         ret = stm32_cryp_gcm_init(cryp, cfg);
 592
 593                 if (ret)
 594                         return ret;
 595
 596                 break;
 597
 598         case CR_DES_CBC:
 599         case CR_TDES_CBC:
 600         case CR_AES_CBC:
 601         case CR_AES_CTR:
 602                 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv);
 603                 break;
 604
 605         default:
 606                 break;
 607         }
 608
 609         /* Enable now */
 610         cfg |= CR_CRYPEN;
 611
 612         stm32_cryp_write(cryp, CRYP_CR, cfg);
 613
 614         return 0;
 615 }
 616
 617 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
 618 {
 619         if (!err && (is_gcm(cryp) || is_ccm(cryp)))
 620                 /* Phase 4 : output tag */
 621                 err = stm32_cryp_read_auth_tag(cryp);
 622
 623         if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp))))
 624                 stm32_cryp_get_iv(cryp);
 625
 626         pm_runtime_mark_last_busy(cryp->dev);
 627         pm_runtime_put_autosuspend(cryp->dev);
 628
 629         if (is_gcm(cryp) || is_ccm(cryp))
 630                 crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
 631         else
 632                 crypto_finalize_skcipher_request(cryp->engine, cryp->req,
 633                                                    err);
 634 }
 635
 636 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
 637 {
 638         /* Enable interrupt and let the IRQ handler do everything */
 639         stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT);
 640
 641         return 0;
 642 }
 643
 644 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);
 645 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
 646                                          void *areq);
 647
 648 static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm)
 649 {
 650         struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 651
 652         crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx));
 653
 654         ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req;
 655         ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req;
 656         ctx->enginectx.op.unprepare_request = NULL;
 657         return 0;
 658 }
 659
 660 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq);
 661 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine,
 662                                        void *areq);
 663
 664 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm)
 665 {
 666         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 667
 668         tfm->reqsize = sizeof(struct stm32_cryp_reqctx);
 669
 670         ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req;
 671         ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req;
 672         ctx->enginectx.op.unprepare_request = NULL;
 673
 674         return 0;
 675 }
 676
 677 static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode)
 678 {
 679         struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
 680                         crypto_skcipher_reqtfm(req));
 681         struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req);
 682         struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
 683
 684         if (!cryp)
 685                 return -ENODEV;
 686
 687         rctx->mode = mode;
 688
 689         return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
 690 }
 691
 692 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)
 693 {
 694         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 695         struct stm32_cryp_reqctx *rctx = aead_request_ctx(req);
 696         struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
 697
 698         if (!cryp)
 699                 return -ENODEV;
 700
 701         rctx->mode = mode;
 702
 703         return crypto_transfer_aead_request_to_engine(cryp->engine, req);
 704 }
 705
 706 static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key,
 707                              unsigned int keylen)
 708 {
 709         struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 710
 711         memcpy(ctx->key, key, keylen);
 712         ctx->keylen = keylen;
 713
 714         return 0;
 715 }
 716
 717 static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 718                                  unsigned int keylen)
 719 {
 720         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 721             keylen != AES_KEYSIZE_256)
 722                 return -EINVAL;
 723         else
 724                 return stm32_cryp_setkey(tfm, key, keylen);
 725 }
 726
 727 static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
 728                                  unsigned int keylen)
 729 {
 730         return verify_skcipher_des_key(tfm, key) ?:
 731                stm32_cryp_setkey(tfm, key, keylen);
 732 }
 733
 734 static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 735                                   unsigned int keylen)
 736 {
 737         return verify_skcipher_des3_key(tfm, key) ?:
 738                stm32_cryp_setkey(tfm, key, keylen);
 739 }
 740
 741 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 742                                       unsigned int keylen)
 743 {
 744         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 745
 746         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 747             keylen != AES_KEYSIZE_256)
 748                 return -EINVAL;
 749
 750         memcpy(ctx->key, key, keylen);
 751         ctx->keylen = keylen;
 752
 753         return 0;
 754 }
 755
 756 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
 757                                           unsigned int authsize)
 758 {
 759         switch (authsize) {
 760         case 4:
 761         case 8:
 762         case 12:
 763         case 13:
 764         case 14:
 765         case 15:
 766         case 16:
 767                 break;
 768         default:
 769                 return -EINVAL;
 770         }
 771
 772         return 0;
 773 }
 774
 775 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
 776                                           unsigned int authsize)
 777 {
 778         switch (authsize) {
 779         case 4:
 780         case 6:
 781         case 8:
 782         case 10:
 783         case 12:
 784         case 14:
 785         case 16:
 786                 break;
 787         default:
 788                 return -EINVAL;
 789         }
 790
 791         return 0;
 792 }
 793
 794 static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)
 795 {
 796         if (req->cryptlen % AES_BLOCK_SIZE)
 797                 return -EINVAL;
 798
 799         if (req->cryptlen == 0)
 800                 return 0;
 801
 802         return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
 803 }
 804
 805 static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)
 806 {
 807         if (req->cryptlen % AES_BLOCK_SIZE)
 808                 return -EINVAL;
 809
 810         if (req->cryptlen == 0)
 811                 return 0;
 812
 813         return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
 814 }
 815
 816 static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)
 817 {
 818         if (req->cryptlen % AES_BLOCK_SIZE)
 819                 return -EINVAL;
 820
 821         if (req->cryptlen == 0)
 822                 return 0;
 823
 824         return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
 825 }
 826
 827 static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)
 828 {
 829         if (req->cryptlen % AES_BLOCK_SIZE)
 830                 return -EINVAL;
 831
 832         if (req->cryptlen == 0)
 833                 return 0;
 834
 835         return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
 836 }
 837
 838 static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)
 839 {
 840         if (req->cryptlen == 0)
 841                 return 0;
 842
 843         return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
 844 }
 845
 846 static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)
 847 {
 848         if (req->cryptlen == 0)
 849                 return 0;
 850
 851         return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
 852 }
 853
 854 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req)
 855 {
 856         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT);
 857 }
 858
 859 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
 860 {
 861         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
 862 }
 863
 864 static inline int crypto_ccm_check_iv(const u8 *iv)
 865 {
 866         /* 2 <= L <= 8, so 1 <= L' <= 7. */
 867         if (iv[0] < 1 || iv[0] > 7)
 868                 return -EINVAL;
 869
 870         return 0;
 871 }
 872
 873 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
 874 {
 875         int err;
 876
 877         err = crypto_ccm_check_iv(req->iv);
 878         if (err)
 879                 return err;
 880
 881         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
 882 }
 883
 884 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
 885 {
 886         int err;
 887
 888         err = crypto_ccm_check_iv(req->iv);
 889         if (err)
 890                 return err;
 891
 892         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
 893 }
 894
 895 static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)
 896 {
 897         if (req->cryptlen % DES_BLOCK_SIZE)
 898                 return -EINVAL;
 899
 900         if (req->cryptlen == 0)
 901                 return 0;
 902
 903         return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
 904 }
 905
 906 static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)
 907 {
 908         if (req->cryptlen % DES_BLOCK_SIZE)
 909                 return -EINVAL;
 910
 911         if (req->cryptlen == 0)
 912                 return 0;
 913
 914         return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
 915 }
 916
 917 static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)
 918 {
 919         if (req->cryptlen % DES_BLOCK_SIZE)
 920                 return -EINVAL;
 921
 922         if (req->cryptlen == 0)
 923                 return 0;
 924
 925         return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
 926 }
 927
 928 static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)
 929 {
 930         if (req->cryptlen % DES_BLOCK_SIZE)
 931                 return -EINVAL;
 932
 933         if (req->cryptlen == 0)
 934                 return 0;
 935
 936         return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
 937 }
 938
 939 static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)
 940 {
 941         if (req->cryptlen % DES_BLOCK_SIZE)
 942                 return -EINVAL;
 943
 944         if (req->cryptlen == 0)
 945                 return 0;
 946
 947         return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
 948 }
 949
 950 static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)
 951 {
 952         if (req->cryptlen % DES_BLOCK_SIZE)
 953                 return -EINVAL;
 954
 955         if (req->cryptlen == 0)
 956                 return 0;
 957
 958         return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
 959 }
 960
 961 static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)
 962 {
 963         if (req->cryptlen % DES_BLOCK_SIZE)
 964                 return -EINVAL;
 965
 966         if (req->cryptlen == 0)
 967                 return 0;
 968
 969         return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
 970 }
 971
 972 static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)
 973 {
 974         if (req->cryptlen % DES_BLOCK_SIZE)
 975                 return -EINVAL;
 976
 977         if (req->cryptlen == 0)
 978                 return 0;
 979
 980         return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
 981 }
 982
 983 static int stm32_cryp_prepare_req(struct skcipher_request *req,
 984                                   struct aead_request *areq)
 985 {
 986         struct stm32_cryp_ctx *ctx;
 987         struct stm32_cryp *cryp;
 988         struct stm32_cryp_reqctx *rctx;
 989         struct scatterlist *in_sg;
 990         int ret;
 991
 992         if (!req && !areq)
 993                 return -EINVAL;
 994
 995         ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
 996                     crypto_aead_ctx(crypto_aead_reqtfm(areq));
 997
 998         cryp = ctx->cryp;
 999
1000         if (!cryp)
1001                 return -ENODEV;
1002
1003         rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
1004         rctx->mode &= FLG_MODE_MASK;
1005
1006         ctx->cryp = cryp;
1007
1008         cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;
1009         cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;
1010         cryp->ctx = ctx;
1011
1012         if (req) {
1013                 cryp->req = req;
1014                 cryp->areq = NULL;
1015                 cryp->header_in = 0;
1016                 cryp->payload_in = req->cryptlen;
1017                 cryp->payload_out = req->cryptlen;
1018                 cryp->authsize = 0;
1019         } else {
1020                 /*
1021                  * Length of input and output data:
1022                  * Encryption case:
1023                  *  INPUT  = AssocData   ||     PlainText
1024                  *          <- assoclen ->  <- cryptlen ->
1025                  *
1026                  *  OUTPUT = AssocData    ||   CipherText   ||      AuthTag
1027                  *          <- assoclen ->  <-- cryptlen -->  <- authsize ->
1028                  *
1029                  * Decryption case:
1030                  *  INPUT  =  AssocData     ||    CipherTex   ||       AuthTag
1031                  *          <- assoclen --->  <---------- cryptlen ---------->
1032                  *
1033                  *  OUTPUT = AssocData    ||               PlainText
1034                  *          <- assoclen ->  <- cryptlen - authsize ->
1035                  */
1036                 cryp->areq = areq;
1037                 cryp->req = NULL;
1038                 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
1039                 if (is_encrypt(cryp)) {
1040                         cryp->payload_in = areq->cryptlen;
1041                         cryp->header_in = areq->assoclen;
1042                         cryp->payload_out = areq->cryptlen;
1043                 } else {
1044                         cryp->payload_in = areq->cryptlen - cryp->authsize;
1045                         cryp->header_in = areq->assoclen;
1046                         cryp->payload_out = cryp->payload_in;
1047                 }
1048         }
1049
1050         in_sg = req ? req->src : areq->src;
1051         scatterwalk_start(&cryp->in_walk, in_sg);
1052
1053         cryp->out_sg = req ? req->dst : areq->dst;
1054         scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1055
1056         if (is_gcm(cryp) || is_ccm(cryp)) {
1057                 /* In output, jump after assoc data */
1058                 scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2);
1059         }
1060
1061         if (is_ctr(cryp))
1062                 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr));
1063
1064         ret = stm32_cryp_hw_init(cryp);
1065         return ret;
1066 }
1067
1068 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
1069                                          void *areq)
1070 {
1071         struct skcipher_request *req = container_of(areq,
1072                                                       struct skcipher_request,
1073                                                       base);
1074
1075         return stm32_cryp_prepare_req(req, NULL);
1076 }
1077
1078 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)
1079 {
1080         struct skcipher_request *req = container_of(areq,
1081                                                       struct skcipher_request,
1082                                                       base);
1083         struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
1084                         crypto_skcipher_reqtfm(req));
1085         struct stm32_cryp *cryp = ctx->cryp;
1086
1087         if (!cryp)
1088                 return -ENODEV;
1089
1090         return stm32_cryp_cpu_start(cryp);
1091 }
1092
1093 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq)
1094 {
1095         struct aead_request *req = container_of(areq, struct aead_request,
1096                                                 base);
1097
1098         return stm32_cryp_prepare_req(NULL, req);
1099 }
1100
1101 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
1102 {
1103         struct aead_request *req = container_of(areq, struct aead_request,
1104                                                 base);
1105         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1106         struct stm32_cryp *cryp = ctx->cryp;
1107
1108         if (!cryp)
1109                 return -ENODEV;
1110
1111         if (unlikely(!cryp->payload_in && !cryp->header_in)) {
1112                 /* No input data to process: get tag and finish */
1113                 stm32_cryp_finish_req(cryp, 0);
1114                 return 0;
1115         }
1116
1117         return stm32_cryp_cpu_start(cryp);
1118 }
1119
1120 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
1121 {
1122         u32 cfg, size_bit;
1123         unsigned int i;
1124         int ret = 0;
1125
1126         /* Update Config */
1127         cfg = stm32_cryp_read(cryp, CRYP_CR);
1128
1129         cfg &= ~CR_PH_MASK;
1130         cfg |= CR_PH_FINAL;
1131         cfg &= ~CR_DEC_NOT_ENC;
1132         cfg |= CR_CRYPEN;
1133
1134         stm32_cryp_write(cryp, CRYP_CR, cfg);
1135
1136         if (is_gcm(cryp)) {
1137                 /* GCM: write aad and payload size (in bits) */
1138                 size_bit = cryp->areq->assoclen * 8;
1139                 if (cryp->caps->swap_final)
1140                         size_bit = (__force u32)cpu_to_be32(size_bit);
1141
1142                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1143                 stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1144
1145                 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
1146                                 cryp->areq->cryptlen - cryp->authsize;
1147                 size_bit *= 8;
1148                 if (cryp->caps->swap_final)
1149                         size_bit = (__force u32)cpu_to_be32(size_bit);
1150
1151                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1152                 stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1153         } else {
1154                 /* CCM: write CTR0 */
1155                 u32 iv32[AES_BLOCK_32];
1156                 u8 *iv = (u8 *)iv32;
1157                 __be32 *biv = (__be32 *)iv32;
1158
1159                 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
1160                 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
1161
1162                 for (i = 0; i < AES_BLOCK_32; i++) {
1163                         u32 xiv = iv32[i];
1164
1165                         if (!cryp->caps->padding_wa)
1166                                 xiv = be32_to_cpu(biv[i]);
1167                         stm32_cryp_write(cryp, CRYP_DIN, xiv);
1168                 }
1169         }
1170
1171         /* Wait for output data */
1172         ret = stm32_cryp_wait_output(cryp);
1173         if (ret) {
1174                 dev_err(cryp->dev, "Timeout (read tag)\n");
1175                 return ret;
1176         }
1177
1178         if (is_encrypt(cryp)) {
1179                 u32 out_tag[AES_BLOCK_32];
1180
1181                 /* Get and write tag */
1182                 for (i = 0; i < AES_BLOCK_32; i++)
1183                         out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1184
1185                 scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1);
1186         } else {
1187                 /* Get and check tag */
1188                 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
1189
1190                 scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0);
1191
1192                 for (i = 0; i < AES_BLOCK_32; i++)
1193                         out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1194
1195                 if (crypto_memneq(in_tag, out_tag, cryp->authsize))
1196                         ret = -EBADMSG;
1197         }
1198
1199         /* Disable cryp */
1200         cfg &= ~CR_CRYPEN;
1201         stm32_cryp_write(cryp, CRYP_CR, cfg);
1202
1203         return ret;
1204 }
1205
1206 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
1207 {
1208         u32 cr;
1209
1210         if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) {
1211                 /*
1212                  * In this case, we need to increment manually the ctr counter,
1213                  * as HW doesn't handle the U32 carry.
1214                  */
1215                 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr));
1216
1217                 cr = stm32_cryp_read(cryp, CRYP_CR);
1218                 stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
1219
1220                 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr);
1221
1222                 stm32_cryp_write(cryp, CRYP_CR, cr);
1223         }
1224
1225         /* The IV registers are BE  */
1226         cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
1227         cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
1228         cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
1229         cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
1230 }
1231
1232 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
1233 {
1234         unsigned int i;
1235         u32 block[AES_BLOCK_32];
1236
1237         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
1238                 block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1239
1240         scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
1241                                                              cryp->payload_out), 1);
1242         cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
1243                                    cryp->payload_out);
1244 }
1245
1246 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
1247 {
1248         unsigned int i;
1249         u32 block[AES_BLOCK_32] = {0};
1250
1251         scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize,
1252                                                             cryp->payload_in), 0);
1253         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
1254                 stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1255
1256         cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in);
1257 }
1258
1259 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
1260 {
1261         int err;
1262         u32 cfg, block[AES_BLOCK_32] = {0};
1263         unsigned int i;
1264
1265         /* 'Special workaround' procedure described in the datasheet */
1266
1267         /* a) disable ip */
1268         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1269         cfg = stm32_cryp_read(cryp, CRYP_CR);
1270         cfg &= ~CR_CRYPEN;
1271         stm32_cryp_write(cryp, CRYP_CR, cfg);
1272
1273         /* b) Update IV1R */
1274         stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2);
1275
1276         /* c) change mode to CTR */
1277         cfg &= ~CR_ALGO_MASK;
1278         cfg |= CR_AES_CTR;
1279         stm32_cryp_write(cryp, CRYP_CR, cfg);
1280
1281         /* a) enable IP */
1282         cfg |= CR_CRYPEN;
1283         stm32_cryp_write(cryp, CRYP_CR, cfg);
1284
1285         /* b) pad and write the last block */
1286         stm32_cryp_irq_write_block(cryp);
1287         /* wait end of process */
1288         err = stm32_cryp_wait_output(cryp);
1289         if (err) {
1290                 dev_err(cryp->dev, "Timeout (write gcm last data)\n");
1291                 return stm32_cryp_finish_req(cryp, err);
1292         }
1293
1294         /* c) get and store encrypted data */
1295         /*
1296          * Same code as stm32_cryp_irq_read_data(), but we want to store
1297          * block value
1298          */
1299         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
1300                 block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1301
1302         scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
1303                                                              cryp->payload_out), 1);
1304         cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
1305                                    cryp->payload_out);
1306
1307         /* d) change mode back to AES GCM */
1308         cfg &= ~CR_ALGO_MASK;
1309         cfg |= CR_AES_GCM;
1310         stm32_cryp_write(cryp, CRYP_CR, cfg);
1311
1312         /* e) change phase to Final */
1313         cfg &= ~CR_PH_MASK;
1314         cfg |= CR_PH_FINAL;
1315         stm32_cryp_write(cryp, CRYP_CR, cfg);
1316
1317         /* f) write padded data */
1318         for (i = 0; i < AES_BLOCK_32; i++)
1319                 stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1320
1321         /* g) Empty fifo out */
1322         err = stm32_cryp_wait_output(cryp);
1323         if (err) {
1324                 dev_err(cryp->dev, "Timeout (write gcm padded data)\n");
1325                 return stm32_cryp_finish_req(cryp, err);
1326         }
1327
1328         for (i = 0; i < AES_BLOCK_32; i++)
1329                 stm32_cryp_read(cryp, CRYP_DOUT);
1330
1331         /* h) run the he normal Final phase */
1332         stm32_cryp_finish_req(cryp, 0);
1333 }
1334
1335 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
1336 {
1337         u32 cfg;
1338
1339         /* disable ip, set NPBLB and reneable ip */
1340         cfg = stm32_cryp_read(cryp, CRYP_CR);
1341         cfg &= ~CR_CRYPEN;
1342         stm32_cryp_write(cryp, CRYP_CR, cfg);
1343
1344         cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT;
1345         cfg |= CR_CRYPEN;
1346         stm32_cryp_write(cryp, CRYP_CR, cfg);
1347 }
1348
1349 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
1350 {
1351         int err = 0;
1352         u32 cfg, iv1tmp;
1353         u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32];
1354         u32 block[AES_BLOCK_32] = {0};
1355         unsigned int i;
1356
1357         /* 'Special workaround' procedure described in the datasheet */
1358
1359         /* a) disable ip */
1360         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1361
1362         cfg = stm32_cryp_read(cryp, CRYP_CR);
1363         cfg &= ~CR_CRYPEN;
1364         stm32_cryp_write(cryp, CRYP_CR, cfg);
1365
1366         /* b) get IV1 from CRYP_CSGCMCCM7 */
1367         iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4);
1368
1369         /* c) Load CRYP_CSGCMCCMxR */
1370         for (i = 0; i < ARRAY_SIZE(cstmp1); i++)
1371                 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1372
1373         /* d) Write IV1R */
1374         stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp);
1375
1376         /* e) change mode to CTR */
1377         cfg &= ~CR_ALGO_MASK;
1378         cfg |= CR_AES_CTR;
1379         stm32_cryp_write(cryp, CRYP_CR, cfg);
1380
1381         /* a) enable IP */
1382         cfg |= CR_CRYPEN;
1383         stm32_cryp_write(cryp, CRYP_CR, cfg);
1384
1385         /* b) pad and write the last block */
1386         stm32_cryp_irq_write_block(cryp);
1387         /* wait end of process */
1388         err = stm32_cryp_wait_output(cryp);
1389         if (err) {
1390                 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1391                 return stm32_cryp_finish_req(cryp, err);
1392         }
1393
1394         /* c) get and store decrypted data */
1395         /*
1396          * Same code as stm32_cryp_irq_read_data(), but we want to store
1397          * block value
1398          */
1399         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
1400                 block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1401
1402         scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
1403                                                              cryp->payload_out), 1);
1404         cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out);
1405
1406         /* d) Load again CRYP_CSGCMCCMxR */
1407         for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
1408                 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1409
1410         /* e) change mode back to AES CCM */
1411         cfg &= ~CR_ALGO_MASK;
1412         cfg |= CR_AES_CCM;
1413         stm32_cryp_write(cryp, CRYP_CR, cfg);
1414
1415         /* f) change phase to header */
1416         cfg &= ~CR_PH_MASK;
1417         cfg |= CR_PH_HEADER;
1418         stm32_cryp_write(cryp, CRYP_CR, cfg);
1419
1420         /* g) XOR and write padded data */
1421         for (i = 0; i < ARRAY_SIZE(block); i++) {
1422                 block[i] ^= cstmp1[i];
1423                 block[i] ^= cstmp2[i];
1424                 stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1425         }
1426
1427         /* h) wait for completion */
1428         err = stm32_cryp_wait_busy(cryp);
1429         if (err)
1430                 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1431
1432         /* i) run the he normal Final phase */
1433         stm32_cryp_finish_req(cryp, err);
1434 }
1435
1436 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
1437 {
1438         if (unlikely(!cryp->payload_in)) {
1439                 dev_warn(cryp->dev, "No more data to process\n");
1440                 return;
1441         }
1442
1443         if (unlikely(cryp->payload_in < AES_BLOCK_SIZE &&
1444                      (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
1445                      is_encrypt(cryp))) {
1446                 /* Padding for AES GCM encryption */
1447                 if (cryp->caps->padding_wa) {
1448                         /* Special case 1 */
1449                         stm32_cryp_irq_write_gcm_padded_data(cryp);
1450                         return;
1451                 }
1452
1453                 /* Setting padding bytes (NBBLB) */
1454                 stm32_cryp_irq_set_npblb(cryp);
1455         }
1456
1457         if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) &&
1458                      (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
1459                      is_decrypt(cryp))) {
1460                 /* Padding for AES CCM decryption */
1461                 if (cryp->caps->padding_wa) {
1462                         /* Special case 2 */
1463                         stm32_cryp_irq_write_ccm_padded_data(cryp);
1464                         return;
1465                 }
1466
1467                 /* Setting padding bytes (NBBLB) */
1468                 stm32_cryp_irq_set_npblb(cryp);
1469         }
1470
1471         if (is_aes(cryp) && is_ctr(cryp))
1472                 stm32_cryp_check_ctr_counter(cryp);
1473
1474         stm32_cryp_irq_write_block(cryp);
1475 }
1476
1477 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp)
1478 {
1479         unsigned int i;
1480         u32 block[AES_BLOCK_32] = {0};
1481         size_t written;
1482
1483         written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in);
1484
1485         scatterwalk_copychunks(block, &cryp->in_walk, written, 0);
1486         for (i = 0; i < AES_BLOCK_32; i++)
1487                 stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1488
1489         cryp->header_in -= written;
1490
1491         stm32_crypt_gcmccm_end_header(cryp);
1492 }
1493
1494 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
1495 {
1496         struct stm32_cryp *cryp = arg;
1497         u32 ph;
1498         u32 it_mask = stm32_cryp_read(cryp, CRYP_IMSCR);
1499
1500         if (cryp->irq_status & MISR_OUT)
1501                 /* Output FIFO IRQ: read data */
1502                 stm32_cryp_irq_read_data(cryp);
1503
1504         if (cryp->irq_status & MISR_IN) {
1505                 if (is_gcm(cryp) || is_ccm(cryp)) {
1506                         ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1507                         if (unlikely(ph == CR_PH_HEADER))
1508                                 /* Write Header */
1509                                 stm32_cryp_irq_write_gcmccm_header(cryp);
1510                         else
1511                                 /* Input FIFO IRQ: write data */
1512                                 stm32_cryp_irq_write_data(cryp);
1513                         if (is_gcm(cryp))
1514                                 cryp->gcm_ctr++;
1515                 } else {
1516                         /* Input FIFO IRQ: write data */
1517                         stm32_cryp_irq_write_data(cryp);
1518                 }
1519         }
1520
1521         /* Mask useless interrupts */
1522         if (!cryp->payload_in && !cryp->header_in)
1523                 it_mask &= ~IMSCR_IN;
1524         if (!cryp->payload_out)
1525                 it_mask &= ~IMSCR_OUT;
1526         stm32_cryp_write(cryp, CRYP_IMSCR, it_mask);
1527
1528         if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out)
1529                 stm32_cryp_finish_req(cryp, 0);
1530
1531         return IRQ_HANDLED;
1532 }
1533
1534 static irqreturn_t stm32_cryp_irq(int irq, void *arg)
1535 {
1536         struct stm32_cryp *cryp = arg;
1537
1538         cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR);
1539
1540         return IRQ_WAKE_THREAD;
1541 }
1542
1543 static struct skcipher_alg crypto_algs[] = {
1544 {
1545         .base.cra_name          = "ecb(aes)",
1546         .base.cra_driver_name   = "stm32-ecb-aes",
1547         .base.cra_priority      = 200,
1548         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1549         .base.cra_blocksize     = AES_BLOCK_SIZE,
1550         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1551         .base.cra_alignmask     = 0,
1552         .base.cra_module        = THIS_MODULE,
1553
1554         .init                   = stm32_cryp_init_tfm,
1555         .min_keysize            = AES_MIN_KEY_SIZE,
1556         .max_keysize            = AES_MAX_KEY_SIZE,
1557         .setkey                 = stm32_cryp_aes_setkey,
1558         .encrypt                = stm32_cryp_aes_ecb_encrypt,
1559         .decrypt                = stm32_cryp_aes_ecb_decrypt,
1560 },
1561 {
1562         .base.cra_name          = "cbc(aes)",
1563         .base.cra_driver_name   = "stm32-cbc-aes",
1564         .base.cra_priority      = 200,
1565         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1566         .base.cra_blocksize     = AES_BLOCK_SIZE,
1567         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1568         .base.cra_alignmask     = 0,
1569         .base.cra_module        = THIS_MODULE,
1570
1571         .init                   = stm32_cryp_init_tfm,
1572         .min_keysize            = AES_MIN_KEY_SIZE,
1573         .max_keysize            = AES_MAX_KEY_SIZE,
1574         .ivsize                 = AES_BLOCK_SIZE,
1575         .setkey                 = stm32_cryp_aes_setkey,
1576         .encrypt                = stm32_cryp_aes_cbc_encrypt,
1577         .decrypt                = stm32_cryp_aes_cbc_decrypt,
1578 },
1579 {
1580         .base.cra_name          = "ctr(aes)",
1581         .base.cra_driver_name   = "stm32-ctr-aes",
1582         .base.cra_priority      = 200,
1583         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1584         .base.cra_blocksize     = 1,
1585         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1586         .base.cra_alignmask     = 0,
1587         .base.cra_module        = THIS_MODULE,
1588
1589         .init                   = stm32_cryp_init_tfm,
1590         .min_keysize            = AES_MIN_KEY_SIZE,
1591         .max_keysize            = AES_MAX_KEY_SIZE,
1592         .ivsize                 = AES_BLOCK_SIZE,
1593         .setkey                 = stm32_cryp_aes_setkey,
1594         .encrypt                = stm32_cryp_aes_ctr_encrypt,
1595         .decrypt                = stm32_cryp_aes_ctr_decrypt,
1596 },
1597 {
1598         .base.cra_name          = "ecb(des)",
1599         .base.cra_driver_name   = "stm32-ecb-des",
1600         .base.cra_priority      = 200,
1601         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1602         .base.cra_blocksize     = DES_BLOCK_SIZE,
1603         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1604         .base.cra_alignmask     = 0,
1605         .base.cra_module        = THIS_MODULE,
1606
1607         .init                   = stm32_cryp_init_tfm,
1608         .min_keysize            = DES_BLOCK_SIZE,
1609         .max_keysize            = DES_BLOCK_SIZE,
1610         .setkey                 = stm32_cryp_des_setkey,
1611         .encrypt                = stm32_cryp_des_ecb_encrypt,
1612         .decrypt                = stm32_cryp_des_ecb_decrypt,
1613 },
1614 {
1615         .base.cra_name          = "cbc(des)",
1616         .base.cra_driver_name   = "stm32-cbc-des",
1617         .base.cra_priority      = 200,
1618         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1619         .base.cra_blocksize     = DES_BLOCK_SIZE,
1620         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1621         .base.cra_alignmask     = 0,
1622         .base.cra_module        = THIS_MODULE,
1623
1624         .init                   = stm32_cryp_init_tfm,
1625         .min_keysize            = DES_BLOCK_SIZE,
1626         .max_keysize            = DES_BLOCK_SIZE,
1627         .ivsize                 = DES_BLOCK_SIZE,
1628         .setkey                 = stm32_cryp_des_setkey,
1629         .encrypt                = stm32_cryp_des_cbc_encrypt,
1630         .decrypt                = stm32_cryp_des_cbc_decrypt,
1631 },
1632 {
1633         .base.cra_name          = "ecb(des3_ede)",
1634         .base.cra_driver_name   = "stm32-ecb-des3",
1635         .base.cra_priority      = 200,
1636         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1637         .base.cra_blocksize     = DES_BLOCK_SIZE,
1638         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1639         .base.cra_alignmask     = 0,
1640         .base.cra_module        = THIS_MODULE,
1641
1642         .init                   = stm32_cryp_init_tfm,
1643         .min_keysize            = 3 * DES_BLOCK_SIZE,
1644         .max_keysize            = 3 * DES_BLOCK_SIZE,
1645         .setkey                 = stm32_cryp_tdes_setkey,
1646         .encrypt                = stm32_cryp_tdes_ecb_encrypt,
1647         .decrypt                = stm32_cryp_tdes_ecb_decrypt,
1648 },
1649 {
1650         .base.cra_name          = "cbc(des3_ede)",
1651         .base.cra_driver_name   = "stm32-cbc-des3",
1652         .base.cra_priority      = 200,
1653         .base.cra_flags         = CRYPTO_ALG_ASYNC,
1654         .base.cra_blocksize     = DES_BLOCK_SIZE,
1655         .base.cra_ctxsize       = sizeof(struct stm32_cryp_ctx),
1656         .base.cra_alignmask     = 0,
1657         .base.cra_module        = THIS_MODULE,
1658
1659         .init                   = stm32_cryp_init_tfm,
1660         .min_keysize            = 3 * DES_BLOCK_SIZE,
1661         .max_keysize            = 3 * DES_BLOCK_SIZE,
1662         .ivsize                 = DES_BLOCK_SIZE,
1663         .setkey                 = stm32_cryp_tdes_setkey,
1664         .encrypt                = stm32_cryp_tdes_cbc_encrypt,
1665         .decrypt                = stm32_cryp_tdes_cbc_decrypt,
1666 },
1667 };
1668
1669 static struct aead_alg aead_algs[] = {
1670 {
1671         .setkey         = stm32_cryp_aes_aead_setkey,
1672         .setauthsize    = stm32_cryp_aes_gcm_setauthsize,
1673         .encrypt        = stm32_cryp_aes_gcm_encrypt,
1674         .decrypt        = stm32_cryp_aes_gcm_decrypt,
1675         .init           = stm32_cryp_aes_aead_init,
1676         .ivsize         = 12,
1677         .maxauthsize    = AES_BLOCK_SIZE,
1678
1679         .base = {
1680                 .cra_name               = "gcm(aes)",
1681                 .cra_driver_name        = "stm32-gcm-aes",
1682                 .cra_priority           = 200,
1683                 .cra_flags              = CRYPTO_ALG_ASYNC,
1684                 .cra_blocksize          = 1,
1685                 .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1686                 .cra_alignmask          = 0,
1687                 .cra_module             = THIS_MODULE,
1688         },
1689 },
1690 {
1691         .setkey         = stm32_cryp_aes_aead_setkey,
1692         .setauthsize    = stm32_cryp_aes_ccm_setauthsize,
1693         .encrypt        = stm32_cryp_aes_ccm_encrypt,
1694         .decrypt        = stm32_cryp_aes_ccm_decrypt,
1695         .init           = stm32_cryp_aes_aead_init,
1696         .ivsize         = AES_BLOCK_SIZE,
1697         .maxauthsize    = AES_BLOCK_SIZE,
1698
1699         .base = {
1700                 .cra_name               = "ccm(aes)",
1701                 .cra_driver_name        = "stm32-ccm-aes",
1702                 .cra_priority           = 200,
1703                 .cra_flags              = CRYPTO_ALG_ASYNC,
1704                 .cra_blocksize          = 1,
1705                 .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1706                 .cra_alignmask          = 0,
1707                 .cra_module             = THIS_MODULE,
1708         },
1709 },
1710 };
1711
1712 static const struct stm32_cryp_caps f7_data = {
1713         .swap_final = true,
1714         .padding_wa = true,
1715 };
1716
1717 static const struct stm32_cryp_caps mp1_data = {
1718         .swap_final = false,
1719         .padding_wa = false,
1720 };
1721
1722 static const struct of_device_id stm32_dt_ids[] = {
1723         { .compatible = "st,stm32f756-cryp", .data = &f7_data},
1724         { .compatible = "st,stm32mp1-cryp", .data = &mp1_data},
1725         {},
1726 };
1727 MODULE_DEVICE_TABLE(of, stm32_dt_ids);
1728
1729 static int stm32_cryp_probe(struct platform_device *pdev)
1730 {
1731         struct device *dev = &pdev->dev;
1732         struct stm32_cryp *cryp;
1733         struct reset_control *rst;
1734         int irq, ret;
1735
1736         cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);
1737         if (!cryp)
1738                 return -ENOMEM;
1739
1740         cryp->caps = of_device_get_match_data(dev);
1741         if (!cryp->caps)
1742                 return -ENODEV;
1743
1744         cryp->dev = dev;
1745
1746         cryp->regs = devm_platform_ioremap_resource(pdev, 0);
1747         if (IS_ERR(cryp->regs))
1748                 return PTR_ERR(cryp->regs);
1749
1750         irq = platform_get_irq(pdev, 0);
1751         if (irq < 0)
1752                 return irq;
1753
1754         ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
1755                                         stm32_cryp_irq_thread, IRQF_ONESHOT,
1756                                         dev_name(dev), cryp);
1757         if (ret) {
1758                 dev_err(dev, "Cannot grab IRQ\n");
1759                 return ret;
1760         }
1761
1762         cryp->clk = devm_clk_get(dev, NULL);
1763         if (IS_ERR(cryp->clk)) {
1764                 dev_err(dev, "Could not get clock\n");
1765                 return PTR_ERR(cryp->clk);
1766         }
1767
1768         ret = clk_prepare_enable(cryp->clk);
1769         if (ret) {
1770                 dev_err(cryp->dev, "Failed to enable clock\n");
1771                 return ret;
1772         }
1773
1774         pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);
1775         pm_runtime_use_autosuspend(dev);
1776
1777         pm_runtime_get_noresume(dev);
1778         pm_runtime_set_active(dev);
1779         pm_runtime_enable(dev);
1780
1781         rst = devm_reset_control_get(dev, NULL);
1782         if (!IS_ERR(rst)) {
1783                 reset_control_assert(rst);
1784                 udelay(2);
1785                 reset_control_deassert(rst);
1786         }
1787
1788         platform_set_drvdata(pdev, cryp);
1789
1790         spin_lock(&cryp_list.lock);
1791         list_add(&cryp->list, &cryp_list.dev_list);
1792         spin_unlock(&cryp_list.lock);
1793
1794         /* Initialize crypto engine */
1795         cryp->engine = crypto_engine_alloc_init(dev, 1);
1796         if (!cryp->engine) {
1797                 dev_err(dev, "Could not init crypto engine\n");
1798                 ret = -ENOMEM;
1799                 goto err_engine1;
1800         }
1801
1802         ret = crypto_engine_start(cryp->engine);
1803         if (ret) {
1804                 dev_err(dev, "Could not start crypto engine\n");
1805                 goto err_engine2;
1806         }
1807
1808         ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
1809         if (ret) {
1810                 dev_err(dev, "Could not register algs\n");
1811                 goto err_algs;
1812         }
1813
1814         ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1815         if (ret)
1816                 goto err_aead_algs;
1817
1818         dev_info(dev, "Initialized\n");
1819
1820         pm_runtime_put_sync(dev);
1821
1822         return 0;
1823
1824 err_aead_algs:
1825         crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
1826 err_algs:
1827 err_engine2:
1828         crypto_engine_exit(cryp->engine);
1829 err_engine1:
1830         spin_lock(&cryp_list.lock);
1831         list_del(&cryp->list);
1832         spin_unlock(&cryp_list.lock);
1833
1834         pm_runtime_disable(dev);
1835         pm_runtime_put_noidle(dev);
1836
1837         clk_disable_unprepare(cryp->clk);
1838
1839         return ret;
1840 }
1841
1842 static int stm32_cryp_remove(struct platform_device *pdev)
1843 {
1844         struct stm32_cryp *cryp = platform_get_drvdata(pdev);
1845         int ret;
1846
1847         if (!cryp)
1848                 return -ENODEV;
1849
1850         ret = pm_runtime_resume_and_get(cryp->dev);
1851         if (ret < 0)
1852                 return ret;
1853
1854         crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1855         crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
1856
1857         crypto_engine_exit(cryp->engine);
1858
1859         spin_lock(&cryp_list.lock);
1860         list_del(&cryp->list);
1861         spin_unlock(&cryp_list.lock);
1862
1863         pm_runtime_disable(cryp->dev);
1864         pm_runtime_put_noidle(cryp->dev);
1865
1866         clk_disable_unprepare(cryp->clk);
1867
1868         return 0;
1869 }
1870
1871 #ifdef CONFIG_PM
1872 static int stm32_cryp_runtime_suspend(struct device *dev)
1873 {
1874         struct stm32_cryp *cryp = dev_get_drvdata(dev);
1875
1876         clk_disable_unprepare(cryp->clk);
1877
1878         return 0;
1879 }
1880
1881 static int stm32_cryp_runtime_resume(struct device *dev)
1882 {
1883         struct stm32_cryp *cryp = dev_get_drvdata(dev);
1884         int ret;
1885
1886         ret = clk_prepare_enable(cryp->clk);
1887         if (ret) {
1888                 dev_err(cryp->dev, "Failed to prepare_enable clock\n");
1889                 return ret;
1890         }
1891
1892         return 0;
1893 }
1894 #endif
1895
1896 static const struct dev_pm_ops stm32_cryp_pm_ops = {
1897         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1898                                 pm_runtime_force_resume)
1899         SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,
1900                            stm32_cryp_runtime_resume, NULL)
1901 };
1902
1903 static struct platform_driver stm32_cryp_driver = {
1904         .probe  = stm32_cryp_probe,
1905         .remove = stm32_cryp_remove,
1906         .driver = {
1907                 .name           = DRIVER_NAME,
1908                 .pm             = &stm32_cryp_pm_ops,
1909                 .of_match_table = stm32_dt_ids,
1910         },
1911 };
1912
1913 module_platform_driver(stm32_cryp_driver);
1914
1915 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
1916 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");
1917 MODULE_LICENSE("GPL");