drivers/crypto/ccree/cc_driver.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6
   7 #include <linux/crypto.h>
   8 #include <linux/moduleparam.h>
   9 #include <linux/types.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/platform_device.h>
  12 #include <linux/slab.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/of.h>
  15 #include <linux/clk.h>
  16 #include <linux/of_address.h>
  17
  18 #include "cc_driver.h"
  19 #include "cc_request_mgr.h"
  20 #include "cc_buffer_mgr.h"
  21 #include "cc_debugfs.h"
  22 #include "cc_cipher.h"
  23 #include "cc_aead.h"
  24 #include "cc_hash.h"
  25 #include "cc_ivgen.h"
  26 #include "cc_sram_mgr.h"
  27 #include "cc_pm.h"
  28 #include "cc_fips.h"
  29
  30 bool cc_dump_desc;
  31 module_param_named(dump_desc, cc_dump_desc, bool, 0600);
  32 MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid");
  33
  34 bool cc_dump_bytes;
  35 module_param_named(dump_bytes, cc_dump_bytes, bool, 0600);
  36 MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid");
  37
  38 struct cc_hw_data {
  39         char *name;
  40         enum cc_hw_rev rev;
  41         u32 sig;
  42 };
  43
  44 /* Hardware revisions defs. */
  45
  46 static const struct cc_hw_data cc712_hw = {
  47         .name = "712", .rev = CC_HW_REV_712, .sig =  0xDCC71200U
  48 };
  49
  50 static const struct cc_hw_data cc710_hw = {
  51         .name = "710", .rev = CC_HW_REV_710, .sig =  0xDCC63200U
  52 };
  53
  54 static const struct cc_hw_data cc630p_hw = {
  55         .name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U
  56 };
  57
  58 static const struct of_device_id arm_ccree_dev_of_match[] = {
  59         { .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw },
  60         { .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw },
  61         { .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw },
  62         {}
  63 };
  64 MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);
  65
  66 void __dump_byte_array(const char *name, const u8 *buf, size_t len)
  67 {
  68         char prefix[64];
  69
  70         if (!buf)
  71                 return;
  72
  73         snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len);
  74
  75         print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf,
  76                        len, false);
  77 }
  78
  79 static irqreturn_t cc_isr(int irq, void *dev_id)
  80 {
  81         struct cc_drvdata *drvdata = (struct cc_drvdata *)dev_id;
  82         struct device *dev = drvdata_to_dev(drvdata);
  83         u32 irr;
  84         u32 imr;
  85
  86         /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
  87
  88         /* read the interrupt status */
  89         irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
  90         dev_dbg(dev, "Got IRR=0x%08X\n", irr);
  91         if (irr == 0) { /* Probably shared interrupt line */
  92                 dev_err(dev, "Got interrupt with empty IRR\n");
  93                 return IRQ_NONE;
  94         }
  95         imr = cc_ioread(drvdata, CC_REG(HOST_IMR));
  96
  97         /* clear interrupt - must be before processing events */
  98         cc_iowrite(drvdata, CC_REG(HOST_ICR), irr);
  99
 100         drvdata->irq = irr;
 101         /* Completion interrupt - most probable */
 102         if (irr & CC_COMP_IRQ_MASK) {
 103                 /* Mask AXI completion interrupt - will be unmasked in
 104                  * Deferred service handler
 105                  */
 106                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_COMP_IRQ_MASK);
 107                 irr &= ~CC_COMP_IRQ_MASK;
 108                 complete_request(drvdata);
 109         }
 110 #ifdef CONFIG_CRYPTO_FIPS
 111         /* TEE FIPS interrupt */
 112         if (irr & CC_GPR0_IRQ_MASK) {
 113                 /* Mask interrupt - will be unmasked in Deferred service
 114                  * handler
 115                  */
 116                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_GPR0_IRQ_MASK);
 117                 irr &= ~CC_GPR0_IRQ_MASK;
 118                 fips_handler(drvdata);
 119         }
 120 #endif
 121         /* AXI error interrupt */
 122         if (irr & CC_AXI_ERR_IRQ_MASK) {
 123                 u32 axi_err;
 124
 125                 /* Read the AXI error ID */
 126                 axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR));
 127                 dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n",
 128                         axi_err);
 129
 130                 irr &= ~CC_AXI_ERR_IRQ_MASK;
 131         }
 132
 133         if (irr) {
 134                 dev_dbg_ratelimited(dev, "IRR includes unknown cause bits (0x%08X)\n",
 135                                     irr);
 136                 /* Just warning */
 137         }
 138
 139         return IRQ_HANDLED;
 140 }
 141
 142 int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
 143 {
 144         unsigned int val, cache_params;
 145         struct device *dev = drvdata_to_dev(drvdata);
 146
 147         /* Unmask all AXI interrupt sources AXI_CFG1 register */
 148         val = cc_ioread(drvdata, CC_REG(AXIM_CFG));
 149         cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK);
 150         dev_dbg(dev, "AXIM_CFG=0x%08X\n",
 151                 cc_ioread(drvdata, CC_REG(AXIM_CFG)));
 152
 153         /* Clear all pending interrupts */
 154         val = cc_ioread(drvdata, CC_REG(HOST_IRR));
 155         dev_dbg(dev, "IRR=0x%08X\n", val);
 156         cc_iowrite(drvdata, CC_REG(HOST_ICR), val);
 157
 158         /* Unmask relevant interrupt cause */
 159         val = CC_COMP_IRQ_MASK | CC_AXI_ERR_IRQ_MASK;
 160
 161         if (drvdata->hw_rev >= CC_HW_REV_712)
 162                 val |= CC_GPR0_IRQ_MASK;
 163
 164         cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val);
 165
 166         cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0);
 167
 168         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 169
 170         if (is_probe)
 171                 dev_dbg(dev, "Cache params previous: 0x%08X\n", val);
 172
 173         cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
 174         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 175
 176         if (is_probe)
 177                 dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
 178                         val, cache_params);
 179
 180         return 0;
 181 }
 182
 183 static int init_cc_resources(struct platform_device *plat_dev)
 184 {
 185         struct resource *req_mem_cc_regs = NULL;
 186         struct cc_drvdata *new_drvdata;
 187         struct device *dev = &plat_dev->dev;
 188         struct device_node *np = dev->of_node;
 189         u32 signature_val;
 190         u64 dma_mask;
 191         const struct cc_hw_data *hw_rev;
 192         const struct of_device_id *dev_id;
 193         struct clk *clk;
 194         int rc = 0;
 195
 196         new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
 197         if (!new_drvdata)
 198                 return -ENOMEM;
 199
 200         dev_id = of_match_node(arm_ccree_dev_of_match, np);
 201         if (!dev_id)
 202                 return -ENODEV;
 203
 204         hw_rev = (struct cc_hw_data *)dev_id->data;
 205         new_drvdata->hw_rev_name = hw_rev->name;
 206         new_drvdata->hw_rev = hw_rev->rev;
 207
 208         if (hw_rev->rev >= CC_HW_REV_712) {
 209                 new_drvdata->hash_len_sz = HASH_LEN_SIZE_712;
 210                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
 211                 new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712);
 212                 new_drvdata->ver_offset = CC_REG(HOST_VERSION_712);
 213         } else {
 214                 new_drvdata->hash_len_sz = HASH_LEN_SIZE_630;
 215                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
 216                 new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630);
 217                 new_drvdata->ver_offset = CC_REG(HOST_VERSION_630);
 218         }
 219
 220         platform_set_drvdata(plat_dev, new_drvdata);
 221         new_drvdata->plat_dev = plat_dev;
 222
 223         clk = devm_clk_get(dev, NULL);
 224         if (IS_ERR(clk))
 225                 switch (PTR_ERR(clk)) {
 226                 /* Clock is optional so this might be fine */
 227                 case -ENOENT:
 228                         break;
 229
 230                 /* Clock not available, let's try again soon */
 231                 case -EPROBE_DEFER:
 232                         return -EPROBE_DEFER;
 233
 234                 default:
 235                         dev_err(dev, "Error getting clock: %ld\n",
 236                                 PTR_ERR(clk));
 237                         return PTR_ERR(clk);
 238                 }
 239         new_drvdata->clk = clk;
 240
 241         new_drvdata->coherent = of_dma_is_coherent(np);
 242
 243         /* Get device resources */
 244         /* First CC registers space */
 245         req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
 246         /* Map registers space */
 247         new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
 248         if (IS_ERR(new_drvdata->cc_base)) {
 249                 dev_err(dev, "Failed to ioremap registers");
 250                 return PTR_ERR(new_drvdata->cc_base);
 251         }
 252
 253         dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name,
 254                 req_mem_cc_regs);
 255         dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n",
 256                 &req_mem_cc_regs->start, new_drvdata->cc_base);
 257
 258         /* Then IRQ */
 259         new_drvdata->irq = platform_get_irq(plat_dev, 0);
 260         if (new_drvdata->irq < 0) {
 261                 dev_err(dev, "Failed getting IRQ resource\n");
 262                 return new_drvdata->irq;
 263         }
 264
 265         rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
 266                               IRQF_SHARED, "ccree", new_drvdata);
 267         if (rc) {
 268                 dev_err(dev, "Could not register to interrupt %d\n",
 269                         new_drvdata->irq);
 270                 return rc;
 271         }
 272         dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
 273
 274         init_completion(&new_drvdata->hw_queue_avail);
 275
 276         if (!plat_dev->dev.dma_mask)
 277                 plat_dev->dev.dma_mask = &plat_dev->dev.coherent_dma_mask;
 278
 279         dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
 280         while (dma_mask > 0x7fffffffUL) {
 281                 if (dma_supported(&plat_dev->dev, dma_mask)) {
 282                         rc = dma_set_coherent_mask(&plat_dev->dev, dma_mask);
 283                         if (!rc)
 284                                 break;
 285                 }
 286                 dma_mask >>= 1;
 287         }
 288
 289         if (rc) {
 290                 dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask);
 291                 return rc;
 292         }
 293
 294         rc = cc_clk_on(new_drvdata);
 295         if (rc) {
 296                 dev_err(dev, "Failed to enable clock");
 297                 return rc;
 298         }
 299
 300         /* Verify correct mapping */
 301         signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
 302         if (signature_val != hw_rev->sig) {
 303                 dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
 304                         signature_val, hw_rev->sig);
 305                 rc = -EINVAL;
 306                 goto post_clk_err;
 307         }
 308         dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val);
 309
 310         /* Display HW versions */
 311         dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
 312                  hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset),
 313                  DRV_MODULE_VERSION);
 314
 315         rc = init_cc_regs(new_drvdata, true);
 316         if (rc) {
 317                 dev_err(dev, "init_cc_regs failed\n");
 318                 goto post_clk_err;
 319         }
 320
 321         rc = cc_debugfs_init(new_drvdata);
 322         if (rc) {
 323                 dev_err(dev, "Failed registering debugfs interface\n");
 324                 goto post_regs_err;
 325         }
 326
 327         rc = cc_fips_init(new_drvdata);
 328         if (rc) {
 329                 dev_err(dev, "CC_FIPS_INIT failed 0x%x\n", rc);
 330                 goto post_debugfs_err;
 331         }
 332         rc = cc_sram_mgr_init(new_drvdata);
 333         if (rc) {
 334                 dev_err(dev, "cc_sram_mgr_init failed\n");
 335                 goto post_fips_init_err;
 336         }
 337
 338         new_drvdata->mlli_sram_addr =
 339                 cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE);
 340         if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) {
 341                 dev_err(dev, "Failed to alloc MLLI Sram buffer\n");
 342                 rc = -ENOMEM;
 343                 goto post_sram_mgr_err;
 344         }
 345
 346         rc = cc_req_mgr_init(new_drvdata);
 347         if (rc) {
 348                 dev_err(dev, "cc_req_mgr_init failed\n");
 349                 goto post_sram_mgr_err;
 350         }
 351
 352         rc = cc_buffer_mgr_init(new_drvdata);
 353         if (rc) {
 354                 dev_err(dev, "buffer_mgr_init failed\n");
 355                 goto post_req_mgr_err;
 356         }
 357
 358         rc = cc_pm_init(new_drvdata);
 359         if (rc) {
 360                 dev_err(dev, "ssi_power_mgr_init failed\n");
 361                 goto post_buf_mgr_err;
 362         }
 363
 364         rc = cc_ivgen_init(new_drvdata);
 365         if (rc) {
 366                 dev_err(dev, "cc_ivgen_init failed\n");
 367                 goto post_buf_mgr_err;
 368         }
 369
 370         /* Allocate crypto algs */
 371         rc = cc_cipher_alloc(new_drvdata);
 372         if (rc) {
 373                 dev_err(dev, "cc_cipher_alloc failed\n");
 374                 goto post_ivgen_err;
 375         }
 376
 377         /* hash must be allocated before aead since hash exports APIs */
 378         rc = cc_hash_alloc(new_drvdata);
 379         if (rc) {
 380                 dev_err(dev, "cc_hash_alloc failed\n");
 381                 goto post_cipher_err;
 382         }
 383
 384         rc = cc_aead_alloc(new_drvdata);
 385         if (rc) {
 386                 dev_err(dev, "cc_aead_alloc failed\n");
 387                 goto post_hash_err;
 388         }
 389
 390         /* All set, we can allow autosuspend */
 391         cc_pm_go(new_drvdata);
 392
 393         /* If we got here and FIPS mode is enabled
 394          * it means all FIPS test passed, so let TEE
 395          * know we're good.
 396          */
 397         cc_set_ree_fips_status(new_drvdata, true);
 398
 399         return 0;
 400
 401 post_hash_err:
 402         cc_hash_free(new_drvdata);
 403 post_cipher_err:
 404         cc_cipher_free(new_drvdata);
 405 post_ivgen_err:
 406         cc_ivgen_fini(new_drvdata);
 407 post_buf_mgr_err:
 408          cc_buffer_mgr_fini(new_drvdata);
 409 post_req_mgr_err:
 410         cc_req_mgr_fini(new_drvdata);
 411 post_sram_mgr_err:
 412         cc_sram_mgr_fini(new_drvdata);
 413 post_fips_init_err:
 414         cc_fips_fini(new_drvdata);
 415 post_debugfs_err:
 416         cc_debugfs_fini(new_drvdata);
 417 post_regs_err:
 418         fini_cc_regs(new_drvdata);
 419 post_clk_err:
 420         cc_clk_off(new_drvdata);
 421         return rc;
 422 }
 423
 424 void fini_cc_regs(struct cc_drvdata *drvdata)
 425 {
 426         /* Mask all interrupts */
 427         cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF);
 428 }
 429
 430 static void cleanup_cc_resources(struct platform_device *plat_dev)
 431 {
 432         struct cc_drvdata *drvdata =
 433                 (struct cc_drvdata *)platform_get_drvdata(plat_dev);
 434
 435         cc_aead_free(drvdata);
 436         cc_hash_free(drvdata);
 437         cc_cipher_free(drvdata);
 438         cc_ivgen_fini(drvdata);
 439         cc_pm_fini(drvdata);
 440         cc_buffer_mgr_fini(drvdata);
 441         cc_req_mgr_fini(drvdata);
 442         cc_sram_mgr_fini(drvdata);
 443         cc_fips_fini(drvdata);
 444         cc_debugfs_fini(drvdata);
 445         fini_cc_regs(drvdata);
 446         cc_clk_off(drvdata);
 447 }
 448
 449 int cc_clk_on(struct cc_drvdata *drvdata)
 450 {
 451         struct clk *clk = drvdata->clk;
 452         int rc;
 453
 454         if (IS_ERR(clk))
 455                 /* Not all devices have a clock associated with CCREE  */
 456                 return 0;
 457
 458         rc = clk_prepare_enable(clk);
 459         if (rc)
 460                 return rc;
 461
 462         return 0;
 463 }
 464
 465 void cc_clk_off(struct cc_drvdata *drvdata)
 466 {
 467         struct clk *clk = drvdata->clk;
 468
 469         if (IS_ERR(clk))
 470                 /* Not all devices have a clock associated with CCREE */
 471                 return;
 472
 473         clk_disable_unprepare(clk);
 474 }
 475
 476 static int ccree_probe(struct platform_device *plat_dev)
 477 {
 478         int rc;
 479         struct device *dev = &plat_dev->dev;
 480
 481         /* Map registers space */
 482         rc = init_cc_resources(plat_dev);
 483         if (rc)
 484                 return rc;
 485
 486         dev_info(dev, "ARM ccree device initialized\n");
 487
 488         return 0;
 489 }
 490
 491 static int ccree_remove(struct platform_device *plat_dev)
 492 {
 493         struct device *dev = &plat_dev->dev;
 494
 495         dev_dbg(dev, "Releasing ccree resources...\n");
 496
 497         cleanup_cc_resources(plat_dev);
 498
 499         dev_info(dev, "ARM ccree device terminated\n");
 500
 501         return 0;
 502 }
 503
 504 static struct platform_driver ccree_driver = {
 505         .driver = {
 506                    .name = "ccree",
 507                    .of_match_table = arm_ccree_dev_of_match,
 508 #ifdef CONFIG_PM
 509                    .pm = &ccree_pm,
 510 #endif
 511         },
 512         .probe = ccree_probe,
 513         .remove = ccree_remove,
 514 };
 515
 516 static int __init ccree_init(void)
 517 {
 518         int ret;
 519
 520         cc_hash_global_init();
 521
 522         ret = cc_debugfs_global_init();
 523         if (ret)
 524                 return ret;
 525
 526         return platform_driver_register(&ccree_driver);
 527 }
 528 module_init(ccree_init);
 529
 530 static void __exit ccree_exit(void)
 531 {
 532         platform_driver_unregister(&ccree_driver);
 533         cc_debugfs_global_fini();
 534 }
 535 module_exit(ccree_exit);
 536
 537 /* Module description */
 538 MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
 539 MODULE_VERSION(DRV_MODULE_VERSION);
 540 MODULE_AUTHOR("ARM");
 541 MODULE_LICENSE("GPL v2");