drivers/crypto/inside-secure/safexcel.c

   1 /*
   2  * Copyright (C) 2017 Marvell
   3  *
   4  * Antoine Tenart <antoine.tenart@free-electrons.com>
   5  *
   6  * This file is licensed under the terms of the GNU General Public
   7  * License version 2. This program is licensed "as is" without any
   8  * warranty of any kind, whether express or implied.
   9  */
  10
  11 #include <linux/clk.h>
  12 #include <linux/device.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmapool.h>
  15 #include <linux/firmware.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/module.h>
  18 #include <linux/of_platform.h>
  19 #include <linux/of_irq.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/workqueue.h>
  22
  23 #include <crypto/internal/hash.h>
  24 #include <crypto/internal/skcipher.h>
  25
  26 #include "safexcel.h"
  27
  28 static u32 max_rings = EIP197_MAX_RINGS;
  29 module_param(max_rings, uint, 0644);
  30 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
  31
  32 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
  33 {
  34         u32 val, htable_offset;
  35         int i;
  36
  37         /* Enable the record cache memory access */
  38         val = readl(priv->base + EIP197_CS_RAM_CTRL);
  39         val &= ~EIP197_TRC_ENABLE_MASK;
  40         val |= EIP197_TRC_ENABLE_0;
  41         writel(val, priv->base + EIP197_CS_RAM_CTRL);
  42
  43         /* Clear all ECC errors */
  44         writel(0, priv->base + EIP197_TRC_ECCCTRL);
  45
  46         /*
  47          * Make sure the cache memory is accessible by taking record cache into
  48          * reset.
  49          */
  50         val = readl(priv->base + EIP197_TRC_PARAMS);
  51         val |= EIP197_TRC_PARAMS_SW_RESET;
  52         val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
  53         writel(val, priv->base + EIP197_TRC_PARAMS);
  54
  55         /* Clear all records */
  56         for (i = 0; i < EIP197_CS_RC_MAX; i++) {
  57                 u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
  58
  59                 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
  60                        EIP197_CS_RC_PREV(EIP197_RC_NULL),
  61                        priv->base + offset);
  62
  63                 val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
  64                 if (i == 0)
  65                         val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
  66                 else if (i == EIP197_CS_RC_MAX - 1)
  67                         val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
  68                 writel(val, priv->base + offset + sizeof(u32));
  69         }
  70
  71         /* Clear the hash table entries */
  72         htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE;
  73         for (i = 0; i < 64; i++)
  74                 writel(GENMASK(29, 0),
  75                        priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
  76
  77         /* Disable the record cache memory access */
  78         val = readl(priv->base + EIP197_CS_RAM_CTRL);
  79         val &= ~EIP197_TRC_ENABLE_MASK;
  80         writel(val, priv->base + EIP197_CS_RAM_CTRL);
  81
  82         /* Write head and tail pointers of the record free chain */
  83         val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
  84               EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1);
  85         writel(val, priv->base + EIP197_TRC_FREECHAIN);
  86
  87         /* Configure the record cache #1 */
  88         val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) |
  89               EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX);
  90         writel(val, priv->base + EIP197_TRC_PARAMS2);
  91
  92         /* Configure the record cache #2 */
  93         val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) |
  94               EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
  95               EIP197_TRC_PARAMS_HTABLE_SZ(2);
  96         writel(val, priv->base + EIP197_TRC_PARAMS);
  97 }
  98
  99 static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
 100                                   const struct firmware *fw, u32 ctrl,
 101                                   u32 prog_en)
 102 {
 103         const u32 *data = (const u32 *)fw->data;
 104         u32 val;
 105         int i;
 106
 107         /* Reset the engine to make its program memory accessible */
 108         writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
 109                EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
 110                EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
 111                priv->base + ctrl);
 112
 113         /* Enable access to the program memory */
 114         writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL);
 115
 116         /* Write the firmware */
 117         for (i = 0; i < fw->size / sizeof(u32); i++)
 118                 writel(be32_to_cpu(data[i]),
 119                        priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
 120
 121         /* Disable access to the program memory */
 122         writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL);
 123
 124         /* Release engine from reset */
 125         val = readl(priv->base + ctrl);
 126         val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
 127         writel(val, priv->base + ctrl);
 128 }
 129
 130 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
 131 {
 132         const char *fw_name[] = {"/*(DEBLOBBED)*/", "/*(DEBLOBBED)*/"};
 133         const struct firmware *fw[FW_NB];
 134         int i, j, ret = 0;
 135         u32 val;
 136
 137         for (i = 0; i < FW_NB; i++) {
 138                 ret = reject_firmware(&fw[i], fw_name[i], priv->dev);
 139                 if (ret) {
 140                         dev_err(priv->dev,
 141                                 "Failed to request firmware %s (%d)\n",
 142                                 fw_name[i], ret);
 143                         goto release_fw;
 144                 }
 145          }
 146
 147         /* Clear the scratchpad memory */
 148         val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
 149         val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
 150                EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
 151                EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
 152                EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
 153         writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
 154
 155         memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0,
 156                EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
 157
 158         eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
 159                               EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
 160
 161         eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL,
 162                               EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
 163
 164 release_fw:
 165         for (j = 0; j < i; j++)
 166                 release_firmware(fw[j]);
 167
 168         return ret;
 169 }
 170
 171 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
 172 {
 173         u32 hdw, cd_size_rnd, val;
 174         int i;
 175
 176         hdw = readl(priv->base + EIP197_HIA_OPTIONS);
 177         hdw &= GENMASK(27, 25);
 178         hdw >>= 25;
 179
 180         cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
 181
 182         for (i = 0; i < priv->config.rings; i++) {
 183                 /* ring base address */
 184                 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
 185                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
 186                 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
 187                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
 188
 189                 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
 190                        priv->config.cd_size,
 191                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE);
 192                 writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
 193                        (EIP197_FETCH_COUNT * priv->config.cd_offset),
 194                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
 195
 196                 /* Configure DMA tx control */
 197                 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
 198                 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
 199                 writel(val,
 200                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG);
 201
 202                 /* clear any pending interrupt */
 203                 writel(GENMASK(5, 0),
 204                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT);
 205         }
 206
 207         return 0;
 208 }
 209
 210 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
 211 {
 212         u32 hdw, rd_size_rnd, val;
 213         int i;
 214
 215         hdw = readl(priv->base + EIP197_HIA_OPTIONS);
 216         hdw &= GENMASK(27, 25);
 217         hdw >>= 25;
 218
 219         rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
 220
 221         for (i = 0; i < priv->config.rings; i++) {
 222                 /* ring base address */
 223                 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
 224                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
 225                 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
 226                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
 227
 228                 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
 229                        priv->config.rd_size,
 230                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE);
 231
 232                 writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
 233                        (EIP197_FETCH_COUNT * priv->config.rd_offset),
 234                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
 235
 236                 /* Configure DMA tx control */
 237                 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
 238                 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
 239                 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG;
 240                 writel(val,
 241                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG);
 242
 243                 /* clear any pending interrupt */
 244                 writel(GENMASK(7, 0),
 245                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT);
 246
 247                 /* enable ring interrupt */
 248                 val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
 249                 val |= EIP197_RDR_IRQ(i);
 250                 writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
 251         }
 252
 253         return 0;
 254 }
 255
 256 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 257 {
 258         u32 version, val;
 259         int i, ret;
 260
 261         /* Determine endianess and configure byte swap */
 262         version = readl(priv->base + EIP197_HIA_VERSION);
 263         val = readl(priv->base + EIP197_HIA_MST_CTRL);
 264
 265         if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
 266                 val |= EIP197_MST_CTRL_BYTE_SWAP;
 267         else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
 268                 val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
 269
 270         writel(val, priv->base + EIP197_HIA_MST_CTRL);
 271
 272
 273         /* Configure wr/rd cache values */
 274         writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
 275                EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
 276                priv->base + EIP197_MST_CTRL);
 277
 278         /* Interrupts reset */
 279
 280         /* Disable all global interrupts */
 281         writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL);
 282
 283         /* Clear any pending interrupt */
 284         writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK);
 285
 286         /* Data Fetch Engine configuration */
 287
 288         /* Reset all DFE threads */
 289         writel(EIP197_DxE_THR_CTRL_RESET_PE,
 290                priv->base + EIP197_HIA_DFE_THR_CTRL);
 291
 292         /* Reset HIA input interface arbiter */
 293         writel(EIP197_HIA_RA_PE_CTRL_RESET,
 294                priv->base + EIP197_HIA_RA_PE_CTRL);
 295
 296         /* DMA transfer size to use */
 297         val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
 298         val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
 299         val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
 300         val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
 301         val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
 302         writel(val, priv->base + EIP197_HIA_DFE_CFG);
 303
 304         /* Leave the DFE threads reset state */
 305         writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL);
 306
 307         /* Configure the procesing engine thresholds */
 308         writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
 309               priv->base + EIP197_PE_IN_DBUF_THRES);
 310         writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
 311               priv->base + EIP197_PE_IN_TBUF_THRES);
 312
 313         /* enable HIA input interface arbiter and rings */
 314         writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
 315                priv->base + EIP197_HIA_RA_PE_CTRL);
 316
 317         /* Data Store Engine configuration */
 318
 319         /* Reset all DSE threads */
 320         writel(EIP197_DxE_THR_CTRL_RESET_PE,
 321                priv->base + EIP197_HIA_DSE_THR_CTRL);
 322
 323         /* Wait for all DSE threads to complete */
 324         while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) &
 325                 GENMASK(15, 12)) != GENMASK(15, 12))
 326                 ;
 327
 328         /* DMA transfer size to use */
 329         val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
 330         val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
 331         val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
 332         val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE;
 333         val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
 334         writel(val, priv->base + EIP197_HIA_DSE_CFG);
 335
 336         /* Leave the DSE threads reset state */
 337         writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL);
 338
 339         /* Configure the procesing engine thresholds */
 340         writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
 341                priv->base + EIP197_PE_OUT_DBUF_THRES);
 342
 343         /* Processing Engine configuration */
 344
 345         /* H/W capabilities selection */
 346         val = EIP197_FUNCTION_RSVD;
 347         val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
 348         val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
 349         val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
 350         val |= EIP197_ALG_SHA2;
 351         writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN);
 352
 353         /* Command Descriptor Rings prepare */
 354         for (i = 0; i < priv->config.rings; i++) {
 355                 /* Clear interrupts for this ring */
 356                 writel(GENMASK(31, 0),
 357                        priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i));
 358
 359                 /* Disable external triggering */
 360                 writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
 361
 362                 /* Clear the pending prepared counter */
 363                 writel(EIP197_xDR_PREP_CLR_COUNT,
 364                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT);
 365
 366                 /* Clear the pending processed counter */
 367                 writel(EIP197_xDR_PROC_CLR_COUNT,
 368                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT);
 369
 370                 writel(0,
 371                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR);
 372                 writel(0,
 373                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR);
 374
 375                 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
 376                        priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE);
 377         }
 378
 379         /* Result Descriptor Ring prepare */
 380         for (i = 0; i < priv->config.rings; i++) {
 381                 /* Disable external triggering*/
 382                 writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
 383
 384                 /* Clear the pending prepared counter */
 385                 writel(EIP197_xDR_PREP_CLR_COUNT,
 386                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT);
 387
 388                 /* Clear the pending processed counter */
 389                 writel(EIP197_xDR_PROC_CLR_COUNT,
 390                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT);
 391
 392                 writel(0,
 393                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR);
 394                 writel(0,
 395                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR);
 396
 397                 /* Ring size */
 398                 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
 399                        priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE);
 400         }
 401
 402         /* Enable command descriptor rings */
 403         writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
 404                priv->base + EIP197_HIA_DFE_THR_CTRL);
 405
 406         /* Enable result descriptor rings */
 407         writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
 408                priv->base + EIP197_HIA_DSE_THR_CTRL);
 409
 410         /* Clear any HIA interrupt */
 411         writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK);
 412
 413         eip197_trc_cache_init(priv);
 414
 415         ret = eip197_load_firmwares(priv);
 416         if (ret)
 417                 return ret;
 418
 419         safexcel_hw_setup_cdesc_rings(priv);
 420         safexcel_hw_setup_rdesc_rings(priv);
 421
 422         return 0;
 423 }
 424
 425 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
 426 {
 427         struct crypto_async_request *req, *backlog;
 428         struct safexcel_context *ctx;
 429         struct safexcel_request *request;
 430         int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
 431
 432         priv->ring[ring].need_dequeue = false;
 433
 434         do {
 435                 spin_lock_bh(&priv->ring[ring].queue_lock);
 436                 backlog = crypto_get_backlog(&priv->ring[ring].queue);
 437                 req = crypto_dequeue_request(&priv->ring[ring].queue);
 438                 spin_unlock_bh(&priv->ring[ring].queue_lock);
 439
 440                 if (!req)
 441                         goto finalize;
 442
 443                 request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
 444                 if (!request) {
 445                         spin_lock_bh(&priv->ring[ring].queue_lock);
 446                         crypto_enqueue_request(&priv->ring[ring].queue, req);
 447                         spin_unlock_bh(&priv->ring[ring].queue_lock);
 448
 449                         priv->ring[ring].need_dequeue = true;
 450                         goto finalize;
 451                 }
 452
 453                 ctx = crypto_tfm_ctx(req->tfm);
 454                 ret = ctx->send(req, ring, request, &commands, &results);
 455                 if (ret) {
 456                         kfree(request);
 457                         req->complete(req, ret);
 458                         priv->ring[ring].need_dequeue = true;
 459                         goto finalize;
 460                 }
 461
 462                 if (backlog)
 463                         backlog->complete(backlog, -EINPROGRESS);
 464
 465                 /* In case the send() helper did not issue any command to push
 466                  * to the engine because the input data was cached, continue to
 467                  * dequeue other requests as this is valid and not an error.
 468                  */
 469                 if (!commands && !results) {
 470                         kfree(request);
 471                         continue;
 472                 }
 473
 474                 spin_lock_bh(&priv->ring[ring].egress_lock);
 475                 list_add_tail(&request->list, &priv->ring[ring].list);
 476                 spin_unlock_bh(&priv->ring[ring].egress_lock);
 477
 478                 cdesc += commands;
 479                 rdesc += results;
 480         } while (nreq++ < EIP197_MAX_BATCH_SZ);
 481
 482 finalize:
 483         if (nreq == EIP197_MAX_BATCH_SZ)
 484                 priv->ring[ring].need_dequeue = true;
 485         else if (!nreq)
 486                 return;
 487
 488         spin_lock_bh(&priv->ring[ring].lock);
 489
 490         /* Configure when we want an interrupt */
 491         writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
 492                EIP197_HIA_RDR_THRESH_PROC_PKT(nreq),
 493                priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH);
 494
 495         /* let the RDR know we have pending descriptors */
 496         writel((rdesc * priv->config.rd_offset) << 2,
 497                priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
 498
 499         /* let the CDR know we have pending descriptors */
 500         writel((cdesc * priv->config.cd_offset) << 2,
 501                priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
 502
 503         spin_unlock_bh(&priv->ring[ring].lock);
 504 }
 505
 506 void safexcel_free_context(struct safexcel_crypto_priv *priv,
 507                            struct crypto_async_request *req,
 508                            int result_sz)
 509 {
 510         struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm);
 511
 512         if (ctx->result_dma)
 513                 dma_unmap_single(priv->dev, ctx->result_dma, result_sz,
 514                                  DMA_FROM_DEVICE);
 515
 516         if (ctx->cache) {
 517                 dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz,
 518                                  DMA_TO_DEVICE);
 519                 kfree(ctx->cache);
 520                 ctx->cache = NULL;
 521                 ctx->cache_sz = 0;
 522         }
 523 }
 524
 525 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
 526 {
 527         struct safexcel_command_desc *cdesc;
 528
 529         /* Acknowledge the command descriptors */
 530         do {
 531                 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
 532                 if (IS_ERR(cdesc)) {
 533                         dev_err(priv->dev,
 534                                 "Could not retrieve the command descriptor\n");
 535                         return;
 536                 }
 537         } while (!cdesc->last_seg);
 538 }
 539
 540 void safexcel_inv_complete(struct crypto_async_request *req, int error)
 541 {
 542         struct safexcel_inv_result *result = req->data;
 543
 544         if (error == -EINPROGRESS)
 545                 return;
 546
 547         result->error = error;
 548         complete(&result->completion);
 549 }
 550
 551 int safexcel_invalidate_cache(struct crypto_async_request *async,
 552                               struct safexcel_context *ctx,
 553                               struct safexcel_crypto_priv *priv,
 554                               dma_addr_t ctxr_dma, int ring,
 555                               struct safexcel_request *request)
 556 {
 557         struct safexcel_command_desc *cdesc;
 558         struct safexcel_result_desc *rdesc;
 559         int ret = 0;
 560
 561         spin_lock_bh(&priv->ring[ring].egress_lock);
 562
 563         /* Prepare command descriptor */
 564         cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
 565         if (IS_ERR(cdesc)) {
 566                 ret = PTR_ERR(cdesc);
 567                 goto unlock;
 568         }
 569
 570         cdesc->control_data.type = EIP197_TYPE_EXTENDED;
 571         cdesc->control_data.options = 0;
 572         cdesc->control_data.refresh = 0;
 573         cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
 574
 575         /* Prepare result descriptor */
 576         rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
 577
 578         if (IS_ERR(rdesc)) {
 579                 ret = PTR_ERR(rdesc);
 580                 goto cdesc_rollback;
 581         }
 582
 583         request->req = async;
 584         goto unlock;
 585
 586 cdesc_rollback:
 587         safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 588
 589 unlock:
 590         spin_unlock_bh(&priv->ring[ring].egress_lock);
 591         return ret;
 592 }
 593
 594 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
 595                                                      int ring)
 596 {
 597         struct safexcel_request *sreq;
 598         struct safexcel_context *ctx;
 599         int ret, i, nreq, ndesc = 0;
 600         bool should_complete;
 601
 602         nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
 603         nreq >>= 24;
 604         nreq &= GENMASK(6, 0);
 605         if (!nreq)
 606                 return;
 607
 608         for (i = 0; i < nreq; i++) {
 609                 spin_lock_bh(&priv->ring[ring].egress_lock);
 610                 sreq = list_first_entry(&priv->ring[ring].list,
 611                                         struct safexcel_request, list);
 612                 list_del(&sreq->list);
 613                 spin_unlock_bh(&priv->ring[ring].egress_lock);
 614
 615                 ctx = crypto_tfm_ctx(sreq->req->tfm);
 616                 ndesc = ctx->handle_result(priv, ring, sreq->req,
 617                                            &should_complete, &ret);
 618                 if (ndesc < 0) {
 619                         kfree(sreq);
 620                         dev_err(priv->dev, "failed to handle result (%d)", ndesc);
 621                         return;
 622                 }
 623
 624                 writel(EIP197_xDR_PROC_xD_PKT(1) |
 625                        EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset),
 626                        priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
 627
 628                 if (should_complete) {
 629                         local_bh_disable();
 630                         sreq->req->complete(sreq->req, ret);
 631                         local_bh_enable();
 632                 }
 633
 634                 kfree(sreq);
 635         }
 636 }
 637
 638 static void safexcel_handle_result_work(struct work_struct *work)
 639 {
 640         struct safexcel_work_data *data =
 641                         container_of(work, struct safexcel_work_data, work);
 642         struct safexcel_crypto_priv *priv = data->priv;
 643
 644         safexcel_handle_result_descriptor(priv, data->ring);
 645
 646         if (priv->ring[data->ring].need_dequeue)
 647                 safexcel_dequeue(data->priv, data->ring);
 648 }
 649
 650 struct safexcel_ring_irq_data {
 651         struct safexcel_crypto_priv *priv;
 652         int ring;
 653 };
 654
 655 static irqreturn_t safexcel_irq_ring(int irq, void *data)
 656 {
 657         struct safexcel_ring_irq_data *irq_data = data;
 658         struct safexcel_crypto_priv *priv = irq_data->priv;
 659         int ring = irq_data->ring;
 660         u32 status, stat;
 661
 662         status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
 663         if (!status)
 664                 return IRQ_NONE;
 665
 666         /* RDR interrupts */
 667         if (status & EIP197_RDR_IRQ(ring)) {
 668                 stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
 669
 670                 if (unlikely(stat & EIP197_xDR_ERR)) {
 671                         /*
 672                          * Fatal error, the RDR is unusable and must be
 673                          * reinitialized. This should not happen under
 674                          * normal circumstances.
 675                          */
 676                         dev_err(priv->dev, "RDR: fatal error.");
 677                 } else if (likely(stat & EIP197_xDR_THRESH)) {
 678                         queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work);
 679                 }
 680
 681                 /* ACK the interrupts */
 682                 writel(stat & 0xff,
 683                        priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
 684         }
 685
 686         /* ACK the interrupts */
 687         writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring));
 688
 689         return IRQ_HANDLED;
 690 }
 691
 692 static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
 693                                      irq_handler_t handler,
 694                                      struct safexcel_ring_irq_data *ring_irq_priv)
 695 {
 696         int ret, irq = platform_get_irq_byname(pdev, name);
 697
 698         if (irq < 0) {
 699                 dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
 700                 return irq;
 701         }
 702
 703         ret = devm_request_irq(&pdev->dev, irq, handler, 0,
 704                                dev_name(&pdev->dev), ring_irq_priv);
 705         if (ret) {
 706                 dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
 707                 return ret;
 708         }
 709
 710         return irq;
 711 }
 712
 713 static struct safexcel_alg_template *safexcel_algs[] = {
 714         &safexcel_alg_ecb_aes,
 715         &safexcel_alg_cbc_aes,
 716         &safexcel_alg_sha1,
 717         &safexcel_alg_sha224,
 718         &safexcel_alg_sha256,
 719         &safexcel_alg_hmac_sha1,
 720 };
 721
 722 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
 723 {
 724         int i, j, ret = 0;
 725
 726         for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
 727                 safexcel_algs[i]->priv = priv;
 728
 729                 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 730                         ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
 731                 else
 732                         ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
 733
 734                 if (ret)
 735                         goto fail;
 736         }
 737
 738         return 0;
 739
 740 fail:
 741         for (j = 0; j < i; j++) {
 742                 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 743                         crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
 744                 else
 745                         crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
 746         }
 747
 748         return ret;
 749 }
 750
 751 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
 752 {
 753         int i;
 754
 755         for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
 756                 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 757                         crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
 758                 else
 759                         crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
 760         }
 761 }
 762
 763 static void safexcel_configure(struct safexcel_crypto_priv *priv)
 764 {
 765         u32 val, mask;
 766
 767         val = readl(priv->base + EIP197_HIA_OPTIONS);
 768         val = (val & GENMASK(27, 25)) >> 25;
 769         mask = BIT(val) - 1;
 770
 771         val = readl(priv->base + EIP197_HIA_OPTIONS);
 772         priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
 773
 774         priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
 775         priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
 776
 777         priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
 778         priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
 779 }
 780
 781 static int safexcel_probe(struct platform_device *pdev)
 782 {
 783         struct device *dev = &pdev->dev;
 784         struct resource *res;
 785         struct safexcel_crypto_priv *priv;
 786         int i, ret;
 787
 788         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 789         if (!priv)
 790                 return -ENOMEM;
 791
 792         priv->dev = dev;
 793
 794         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 795         priv->base = devm_ioremap_resource(dev, res);
 796         if (IS_ERR(priv->base)) {
 797                 dev_err(dev, "failed to get resource\n");
 798                 return PTR_ERR(priv->base);
 799         }
 800
 801         priv->clk = devm_clk_get(&pdev->dev, NULL);
 802         if (!IS_ERR(priv->clk)) {
 803                 ret = clk_prepare_enable(priv->clk);
 804                 if (ret) {
 805                         dev_err(dev, "unable to enable clk (%d)\n", ret);
 806                         return ret;
 807                 }
 808         } else {
 809                 /* The clock isn't mandatory */
 810                 if (PTR_ERR(priv->clk) == -EPROBE_DEFER)
 811                         return -EPROBE_DEFER;
 812         }
 813
 814         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
 815         if (ret)
 816                 goto err_clk;
 817
 818         priv->context_pool = dmam_pool_create("safexcel-context", dev,
 819                                               sizeof(struct safexcel_context_record),
 820                                               1, 0);
 821         if (!priv->context_pool) {
 822                 ret = -ENOMEM;
 823                 goto err_clk;
 824         }
 825
 826         safexcel_configure(priv);
 827
 828         for (i = 0; i < priv->config.rings; i++) {
 829                 char irq_name[6] = {0}; /* "ringX\0" */
 830                 char wq_name[9] = {0}; /* "wq_ringX\0" */
 831                 int irq;
 832                 struct safexcel_ring_irq_data *ring_irq;
 833
 834                 ret = safexcel_init_ring_descriptors(priv,
 835                                                      &priv->ring[i].cdr,
 836                                                      &priv->ring[i].rdr);
 837                 if (ret)
 838                         goto err_clk;
 839
 840                 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
 841                 if (!ring_irq) {
 842                         ret = -ENOMEM;
 843                         goto err_clk;
 844                 }
 845
 846                 ring_irq->priv = priv;
 847                 ring_irq->ring = i;
 848
 849                 snprintf(irq_name, 6, "ring%d", i);
 850                 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
 851                                                 ring_irq);
 852                 if (irq < 0) {
 853                         ret = irq;
 854                         goto err_clk;
 855                 }
 856
 857                 priv->ring[i].work_data.priv = priv;
 858                 priv->ring[i].work_data.ring = i;
 859                 INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work);
 860
 861                 snprintf(wq_name, 9, "wq_ring%d", i);
 862                 priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
 863                 if (!priv->ring[i].workqueue) {
 864                         ret = -ENOMEM;
 865                         goto err_clk;
 866                 }
 867
 868                 crypto_init_queue(&priv->ring[i].queue,
 869                                   EIP197_DEFAULT_RING_SIZE);
 870
 871                 INIT_LIST_HEAD(&priv->ring[i].list);
 872                 spin_lock_init(&priv->ring[i].lock);
 873                 spin_lock_init(&priv->ring[i].egress_lock);
 874                 spin_lock_init(&priv->ring[i].queue_lock);
 875         }
 876
 877         platform_set_drvdata(pdev, priv);
 878         atomic_set(&priv->ring_used, 0);
 879
 880         ret = safexcel_hw_init(priv);
 881         if (ret) {
 882                 dev_err(dev, "EIP h/w init failed (%d)\n", ret);
 883                 goto err_clk;
 884         }
 885
 886         ret = safexcel_register_algorithms(priv);
 887         if (ret) {
 888                 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
 889                 goto err_clk;
 890         }
 891
 892         return 0;
 893
 894 err_clk:
 895         clk_disable_unprepare(priv->clk);
 896         return ret;
 897 }
 898
 899
 900 static int safexcel_remove(struct platform_device *pdev)
 901 {
 902         struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
 903         int i;
 904
 905         safexcel_unregister_algorithms(priv);
 906         clk_disable_unprepare(priv->clk);
 907
 908         for (i = 0; i < priv->config.rings; i++)
 909                 destroy_workqueue(priv->ring[i].workqueue);
 910
 911         return 0;
 912 }
 913
 914 static const struct of_device_id safexcel_of_match_table[] = {
 915         { .compatible = "inside-secure,safexcel-eip197" },
 916         {},
 917 };
 918
 919
 920 static struct platform_driver  crypto_safexcel = {
 921         .probe          = safexcel_probe,
 922         .remove         = safexcel_remove,
 923         .driver         = {
 924                 .name   = "crypto-safexcel",
 925                 .of_match_table = safexcel_of_match_table,
 926         },
 927 };
 928 module_platform_driver(crypto_safexcel);
 929
 930 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
 931 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
 932 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
 933 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
 934 MODULE_LICENSE("GPL v2");