hisilicon/sec/sec_drv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for the HiSilicon SEC units found on Hip06 Hip07
   4  *
   5  * Copyright (c) 2016-2017 HiSilicon Limited.
   6  */
   7 #include <linux/acpi.h>
   8 #include <linux/atomic.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-direction.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/io.h>
  14 #include <linux/iommu.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/irq.h>
  17 #include <linux/irqreturn.h>
  18 #include <linux/mm.h>
  19 #include <linux/module.h>
  20 #include <linux/of.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/slab.h>
  23
  24 #include "sec_drv.h"
  25
  26 #define SEC_QUEUE_AR_FROCE_ALLOC                        0
  27 #define SEC_QUEUE_AR_FROCE_NOALLOC                      1
  28 #define SEC_QUEUE_AR_FROCE_DIS                          2
  29
  30 #define SEC_QUEUE_AW_FROCE_ALLOC                        0
  31 #define SEC_QUEUE_AW_FROCE_NOALLOC                      1
  32 #define SEC_QUEUE_AW_FROCE_DIS                          2
  33
  34 /* SEC_ALGSUB registers */
  35 #define SEC_ALGSUB_CLK_EN_REG                           0x03b8
  36 #define SEC_ALGSUB_CLK_DIS_REG                          0x03bc
  37 #define SEC_ALGSUB_CLK_ST_REG                           0x535c
  38 #define SEC_ALGSUB_RST_REQ_REG                          0x0aa8
  39 #define SEC_ALGSUB_RST_DREQ_REG                         0x0aac
  40 #define SEC_ALGSUB_RST_ST_REG                           0x5a54
  41 #define   SEC_ALGSUB_RST_ST_IS_RST                      BIT(0)
  42
  43 #define SEC_ALGSUB_BUILD_RST_REQ_REG                    0x0ab8
  44 #define SEC_ALGSUB_BUILD_RST_DREQ_REG                   0x0abc
  45 #define SEC_ALGSUB_BUILD_RST_ST_REG                     0x5a5c
  46 #define   SEC_ALGSUB_BUILD_RST_ST_IS_RST                BIT(0)
  47
  48 #define SEC_SAA_BASE                                    0x00001000UL
  49
  50 /* SEC_SAA registers */
  51 #define SEC_SAA_CTRL_REG(x)     ((x) * SEC_SAA_ADDR_SIZE)
  52 #define   SEC_SAA_CTRL_GET_QM_EN                        BIT(0)
  53
  54 #define SEC_ST_INTMSK1_REG                              0x0200
  55 #define SEC_ST_RINT1_REG                                0x0400
  56 #define SEC_ST_INTSTS1_REG                              0x0600
  57 #define SEC_BD_MNG_STAT_REG                             0x0800
  58 #define SEC_PARSING_STAT_REG                            0x0804
  59 #define SEC_LOAD_TIME_OUT_CNT_REG                       0x0808
  60 #define SEC_CORE_WORK_TIME_OUT_CNT_REG                  0x080c
  61 #define SEC_BACK_TIME_OUT_CNT_REG                       0x0810
  62 #define SEC_BD1_PARSING_RD_TIME_OUT_CNT_REG             0x0814
  63 #define SEC_BD1_PARSING_WR_TIME_OUT_CNT_REG             0x0818
  64 #define SEC_BD2_PARSING_RD_TIME_OUT_CNT_REG             0x081c
  65 #define SEC_BD2_PARSING_WR_TIME_OUT_CNT_REG             0x0820
  66 #define SEC_SAA_ACC_REG                                 0x083c
  67 #define SEC_BD_NUM_CNT_IN_SEC_REG                       0x0858
  68 #define SEC_LOAD_WORK_TIME_CNT_REG                      0x0860
  69 #define SEC_CORE_WORK_WORK_TIME_CNT_REG                 0x0864
  70 #define SEC_BACK_WORK_TIME_CNT_REG                      0x0868
  71 #define SEC_SAA_IDLE_TIME_CNT_REG                       0x086c
  72 #define SEC_SAA_CLK_CNT_REG                             0x0870
  73
  74 /* SEC_COMMON registers */
  75 #define SEC_CLK_EN_REG                                  0x0000
  76 #define SEC_CTRL_REG                                    0x0004
  77
  78 #define SEC_COMMON_CNT_CLR_CE_REG                       0x0008
  79 #define   SEC_COMMON_CNT_CLR_CE_CLEAR                   BIT(0)
  80 #define   SEC_COMMON_CNT_CLR_CE_SNAP_EN                 BIT(1)
  81
  82 #define SEC_SECURE_CTRL_REG                             0x000c
  83 #define SEC_AXI_CACHE_CFG_REG                           0x0010
  84 #define SEC_AXI_QOS_CFG_REG                             0x0014
  85 #define SEC_IPV4_MASK_TABLE_REG                         0x0020
  86 #define SEC_IPV6_MASK_TABLE_X_REG(x)    (0x0024 + (x) * 4)
  87 #define SEC_FSM_MAX_CNT_REG                             0x0064
  88
  89 #define SEC_CTRL2_REG                                   0x0068
  90 #define   SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M              GENMASK(3, 0)
  91 #define   SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S              0
  92 #define   SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M              GENMASK(6, 4)
  93 #define   SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S              4
  94 #define   SEC_CTRL2_CLK_GATE_EN                         BIT(7)
  95 #define   SEC_CTRL2_ENDIAN_BD                           BIT(8)
  96 #define   SEC_CTRL2_ENDIAN_BD_TYPE                      BIT(9)
  97
  98 #define SEC_CNT_PRECISION_CFG_REG                       0x006c
  99 #define SEC_DEBUG_BD_CFG_REG                            0x0070
 100 #define   SEC_DEBUG_BD_CFG_WB_NORMAL                    BIT(0)
 101 #define   SEC_DEBUG_BD_CFG_WB_EN                        BIT(1)
 102
 103 #define SEC_Q_SIGHT_SEL                                 0x0074
 104 #define SEC_Q_SIGHT_HIS_CLR                             0x0078
 105 #define SEC_Q_VMID_CFG_REG(q)           (0x0100 + (q) * 4)
 106 #define SEC_Q_WEIGHT_CFG_REG(q)         (0x200 + (q) * 4)
 107 #define SEC_STAT_CLR_REG                                0x0a00
 108 #define SEC_SAA_IDLE_CNT_CLR_REG                        0x0a04
 109 #define SEC_QM_CPL_Q_IDBUF_DFX_CFG_REG                  0x0b00
 110 #define SEC_QM_CPL_Q_IDBUF_DFX_RESULT_REG               0x0b04
 111 #define SEC_QM_BD_DFX_CFG_REG                           0x0b08
 112 #define SEC_QM_BD_DFX_RESULT_REG                        0x0b0c
 113 #define SEC_QM_BDID_DFX_RESULT_REG                      0x0b10
 114 #define SEC_QM_BD_DFIFO_STATUS_REG                      0x0b14
 115 #define SEC_QM_BD_DFX_CFG2_REG                          0x0b1c
 116 #define SEC_QM_BD_DFX_RESULT2_REG                       0x0b20
 117 #define SEC_QM_BD_IDFIFO_STATUS_REG                     0x0b18
 118 #define SEC_QM_BD_DFIFO_STATUS2_REG                     0x0b28
 119 #define SEC_QM_BD_IDFIFO_STATUS2_REG                    0x0b2c
 120
 121 #define SEC_HASH_IPV4_MASK                              0xfff00000
 122 #define SEC_MAX_SAA_NUM                                 0xa
 123 #define SEC_SAA_ADDR_SIZE                               0x1000
 124
 125 #define SEC_Q_INIT_REG                                  0x0
 126 #define   SEC_Q_INIT_WO_STAT_CLEAR                      0x2
 127 #define   SEC_Q_INIT_AND_STAT_CLEAR                     0x3
 128
 129 #define SEC_Q_CFG_REG                                   0x8
 130 #define   SEC_Q_CFG_REORDER                             BIT(0)
 131
 132 #define SEC_Q_PROC_NUM_CFG_REG                          0x10
 133 #define SEC_QUEUE_ENB_REG                               0x18
 134
 135 #define SEC_Q_DEPTH_CFG_REG                             0x50
 136 #define   SEC_Q_DEPTH_CFG_DEPTH_M                       GENMASK(11, 0)
 137 #define   SEC_Q_DEPTH_CFG_DEPTH_S                       0
 138
 139 #define SEC_Q_BASE_HADDR_REG                            0x54
 140 #define SEC_Q_BASE_LADDR_REG                            0x58
 141 #define SEC_Q_WR_PTR_REG                                0x5c
 142 #define SEC_Q_OUTORDER_BASE_HADDR_REG                   0x60
 143 #define SEC_Q_OUTORDER_BASE_LADDR_REG                   0x64
 144 #define SEC_Q_OUTORDER_RD_PTR_REG                       0x68
 145 #define SEC_Q_OT_TH_REG                                 0x6c
 146
 147 #define SEC_Q_ARUSER_CFG_REG                            0x70
 148 #define   SEC_Q_ARUSER_CFG_FA                           BIT(0)
 149 #define   SEC_Q_ARUSER_CFG_FNA                          BIT(1)
 150 #define   SEC_Q_ARUSER_CFG_RINVLD                       BIT(2)
 151 #define   SEC_Q_ARUSER_CFG_PKG                          BIT(3)
 152
 153 #define SEC_Q_AWUSER_CFG_REG                            0x74
 154 #define   SEC_Q_AWUSER_CFG_FA                           BIT(0)
 155 #define   SEC_Q_AWUSER_CFG_FNA                          BIT(1)
 156 #define   SEC_Q_AWUSER_CFG_PKG                          BIT(2)
 157
 158 #define SEC_Q_ERR_BASE_HADDR_REG                        0x7c
 159 #define SEC_Q_ERR_BASE_LADDR_REG                        0x80
 160 #define SEC_Q_CFG_VF_NUM_REG                            0x84
 161 #define SEC_Q_SOFT_PROC_PTR_REG                         0x88
 162 #define SEC_Q_FAIL_INT_MSK_REG                          0x300
 163 #define SEC_Q_FLOW_INT_MKS_REG                          0x304
 164 #define SEC_Q_FAIL_RINT_REG                             0x400
 165 #define SEC_Q_FLOW_RINT_REG                             0x404
 166 #define SEC_Q_FAIL_INT_STATUS_REG                       0x500
 167 #define SEC_Q_FLOW_INT_STATUS_REG                       0x504
 168 #define SEC_Q_STATUS_REG                                0x600
 169 #define SEC_Q_RD_PTR_REG                                0x604
 170 #define SEC_Q_PRO_PTR_REG                               0x608
 171 #define SEC_Q_OUTORDER_WR_PTR_REG                       0x60c
 172 #define SEC_Q_OT_CNT_STATUS_REG                         0x610
 173 #define SEC_Q_INORDER_BD_NUM_ST_REG                     0x650
 174 #define SEC_Q_INORDER_GET_FLAG_ST_REG                   0x654
 175 #define SEC_Q_INORDER_ADD_FLAG_ST_REG                   0x658
 176 #define SEC_Q_INORDER_TASK_INT_NUM_LEFT_ST_REG          0x65c
 177 #define SEC_Q_RD_DONE_PTR_REG                           0x660
 178 #define SEC_Q_CPL_Q_BD_NUM_ST_REG                       0x700
 179 #define SEC_Q_CPL_Q_PTR_ST_REG                          0x704
 180 #define SEC_Q_CPL_Q_H_ADDR_ST_REG                       0x708
 181 #define SEC_Q_CPL_Q_L_ADDR_ST_REG                       0x70c
 182 #define SEC_Q_CPL_TASK_INT_NUM_LEFT_ST_REG              0x710
 183 #define SEC_Q_WRR_ID_CHECK_REG                          0x714
 184 #define SEC_Q_CPLQ_FULL_CHECK_REG                       0x718
 185 #define SEC_Q_SUCCESS_BD_CNT_REG                        0x800
 186 #define SEC_Q_FAIL_BD_CNT_REG                           0x804
 187 #define SEC_Q_GET_BD_CNT_REG                            0x808
 188 #define SEC_Q_IVLD_CNT_REG                              0x80c
 189 #define SEC_Q_BD_PROC_GET_CNT_REG                       0x810
 190 #define SEC_Q_BD_PROC_DONE_CNT_REG                      0x814
 191 #define SEC_Q_LAT_CLR_REG                               0x850
 192 #define SEC_Q_PKT_LAT_MAX_REG                           0x854
 193 #define SEC_Q_PKT_LAT_AVG_REG                           0x858
 194 #define SEC_Q_PKT_LAT_MIN_REG                           0x85c
 195 #define SEC_Q_ID_CLR_CFG_REG                            0x900
 196 #define SEC_Q_1ST_BD_ERR_ID_REG                         0x904
 197 #define SEC_Q_1ST_AUTH_FAIL_ID_REG                      0x908
 198 #define SEC_Q_1ST_RD_ERR_ID_REG                         0x90c
 199 #define SEC_Q_1ST_ECC2_ERR_ID_REG                       0x910
 200 #define SEC_Q_1ST_IVLD_ID_REG                           0x914
 201 #define SEC_Q_1ST_BD_WR_ERR_ID_REG                      0x918
 202 #define SEC_Q_1ST_ERR_BD_WR_ERR_ID_REG                  0x91c
 203 #define SEC_Q_1ST_BD_MAC_WR_ERR_ID_REG                  0x920
 204
 205 struct sec_debug_bd_info {
 206 #define SEC_DEBUG_BD_INFO_SOFT_ERR_CHECK_M      GENMASK(22, 0)
 207         u32 soft_err_check;
 208 #define SEC_DEBUG_BD_INFO_HARD_ERR_CHECK_M      GENMASK(9, 0)
 209         u32 hard_err_check;
 210         u32 icv_mac1st_word;
 211 #define SEC_DEBUG_BD_INFO_GET_ID_M              GENMASK(19, 0)
 212         u32 sec_get_id;
 213         /* W4---W15 */
 214         u32 reserv_left[12];
 215 };
 216
 217 struct sec_out_bd_info  {
 218 #define SEC_OUT_BD_INFO_Q_ID_M                  GENMASK(11, 0)
 219 #define SEC_OUT_BD_INFO_ECC_2BIT_ERR            BIT(14)
 220         u16 data;
 221 };
 222
 223 #define SEC_MAX_DEVICES                         8
 224 static struct sec_dev_info *sec_devices[SEC_MAX_DEVICES];
 225 static DEFINE_MUTEX(sec_id_lock);
 226
 227 static int sec_queue_map_io(struct sec_queue *queue)
 228 {
 229         struct device *dev = queue->dev_info->dev;
 230         struct resource *res;
 231
 232         res = platform_get_resource(to_platform_device(dev),
 233                                     IORESOURCE_MEM,
 234                                     2 + queue->queue_id);
 235         if (!res) {
 236                 dev_err(dev, "Failed to get queue %u memory resource\n",
 237                         queue->queue_id);
 238                 return -ENOMEM;
 239         }
 240         queue->regs = ioremap(res->start, resource_size(res));
 241         if (!queue->regs)
 242                 return -ENOMEM;
 243
 244         return 0;
 245 }
 246
 247 static void sec_queue_unmap_io(struct sec_queue *queue)
 248 {
 249          iounmap(queue->regs);
 250 }
 251
 252 static int sec_queue_ar_pkgattr(struct sec_queue *queue, u32 ar_pkg)
 253 {
 254         void __iomem *addr = queue->regs +  SEC_Q_ARUSER_CFG_REG;
 255         u32 regval;
 256
 257         regval = readl_relaxed(addr);
 258         if (ar_pkg)
 259                 regval |= SEC_Q_ARUSER_CFG_PKG;
 260         else
 261                 regval &= ~SEC_Q_ARUSER_CFG_PKG;
 262         writel_relaxed(regval, addr);
 263
 264         return 0;
 265 }
 266
 267 static int sec_queue_aw_pkgattr(struct sec_queue *queue, u32 aw_pkg)
 268 {
 269         void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
 270         u32 regval;
 271
 272         regval = readl_relaxed(addr);
 273         regval |= SEC_Q_AWUSER_CFG_PKG;
 274         writel_relaxed(regval, addr);
 275
 276         return 0;
 277 }
 278
 279 static int sec_clk_en(struct sec_dev_info *info)
 280 {
 281         void __iomem *base = info->regs[SEC_COMMON];
 282         u32 i = 0;
 283
 284         writel_relaxed(0x7, base + SEC_ALGSUB_CLK_EN_REG);
 285         do {
 286                 usleep_range(1000, 10000);
 287                 if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0x7)
 288                         return 0;
 289                 i++;
 290         } while (i < 10);
 291         dev_err(info->dev, "sec clock enable fail!\n");
 292
 293         return -EIO;
 294 }
 295
 296 static int sec_clk_dis(struct sec_dev_info *info)
 297 {
 298         void __iomem *base = info->regs[SEC_COMMON];
 299         u32 i = 0;
 300
 301         writel_relaxed(0x7, base + SEC_ALGSUB_CLK_DIS_REG);
 302         do {
 303                 usleep_range(1000, 10000);
 304                 if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0)
 305                         return 0;
 306                 i++;
 307         } while (i < 10);
 308         dev_err(info->dev, "sec clock disable fail!\n");
 309
 310         return -EIO;
 311 }
 312
 313 static int sec_reset_whole_module(struct sec_dev_info *info)
 314 {
 315         void __iomem *base = info->regs[SEC_COMMON];
 316         bool is_reset, b_is_reset;
 317         u32 i = 0;
 318
 319         writel_relaxed(1, base + SEC_ALGSUB_RST_REQ_REG);
 320         writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_REQ_REG);
 321         while (1) {
 322                 usleep_range(1000, 10000);
 323                 is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
 324                         SEC_ALGSUB_RST_ST_IS_RST;
 325                 b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
 326                         SEC_ALGSUB_BUILD_RST_ST_IS_RST;
 327                 if (is_reset && b_is_reset)
 328                         break;
 329                 i++;
 330                 if (i > 10) {
 331                         dev_err(info->dev, "Reset req failed\n");
 332                         return -EIO;
 333                 }
 334         }
 335
 336         i = 0;
 337         writel_relaxed(1, base + SEC_ALGSUB_RST_DREQ_REG);
 338         writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_DREQ_REG);
 339         while (1) {
 340                 usleep_range(1000, 10000);
 341                 is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
 342                         SEC_ALGSUB_RST_ST_IS_RST;
 343                 b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
 344                         SEC_ALGSUB_BUILD_RST_ST_IS_RST;
 345                 if (!is_reset && !b_is_reset)
 346                         break;
 347
 348                 i++;
 349                 if (i > 10) {
 350                         dev_err(info->dev, "Reset dreq failed\n");
 351                         return -EIO;
 352                 }
 353         }
 354
 355         return 0;
 356 }
 357
 358 static void sec_bd_endian_little(struct sec_dev_info *info)
 359 {
 360         void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
 361         u32 regval;
 362
 363         regval = readl_relaxed(addr);
 364         regval &= ~(SEC_CTRL2_ENDIAN_BD | SEC_CTRL2_ENDIAN_BD_TYPE);
 365         writel_relaxed(regval, addr);
 366 }
 367
 368 /*
 369  * sec_cache_config - configure optimum cache placement
 370  */
 371 static void sec_cache_config(struct sec_dev_info *info)
 372 {
 373         struct iommu_domain *domain;
 374         void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL_REG;
 375
 376         domain = iommu_get_domain_for_dev(info->dev);
 377
 378         /* Check that translation is occurring */
 379         if (domain && (domain->type & __IOMMU_DOMAIN_PAGING))
 380                 writel_relaxed(0x44cf9e, addr);
 381         else
 382                 writel_relaxed(0x4cfd9, addr);
 383 }
 384
 385 static void sec_data_axiwr_otsd_cfg(struct sec_dev_info *info, u32 cfg)
 386 {
 387         void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
 388         u32 regval;
 389
 390         regval = readl_relaxed(addr);
 391         regval &= ~SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
 392         regval |= (cfg << SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S) &
 393                 SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
 394         writel_relaxed(regval, addr);
 395 }
 396
 397 static void sec_data_axird_otsd_cfg(struct sec_dev_info *info, u32 cfg)
 398 {
 399         void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
 400         u32 regval;
 401
 402         regval = readl_relaxed(addr);
 403         regval &= ~SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
 404         regval |= (cfg << SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S) &
 405                 SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
 406         writel_relaxed(regval, addr);
 407 }
 408
 409 static void sec_clk_gate_en(struct sec_dev_info *info, bool clkgate)
 410 {
 411         void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
 412         u32 regval;
 413
 414         regval = readl_relaxed(addr);
 415         if (clkgate)
 416                 regval |= SEC_CTRL2_CLK_GATE_EN;
 417         else
 418                 regval &= ~SEC_CTRL2_CLK_GATE_EN;
 419         writel_relaxed(regval, addr);
 420 }
 421
 422 static void sec_comm_cnt_cfg(struct sec_dev_info *info, bool clr_ce)
 423 {
 424         void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG;
 425         u32 regval;
 426
 427         regval = readl_relaxed(addr);
 428         if (clr_ce)
 429                 regval |= SEC_COMMON_CNT_CLR_CE_CLEAR;
 430         else
 431                 regval &= ~SEC_COMMON_CNT_CLR_CE_CLEAR;
 432         writel_relaxed(regval, addr);
 433 }
 434
 435 static void sec_commsnap_en(struct sec_dev_info *info, bool snap_en)
 436 {
 437         void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG;
 438         u32 regval;
 439
 440         regval = readl_relaxed(addr);
 441         if (snap_en)
 442                 regval |= SEC_COMMON_CNT_CLR_CE_SNAP_EN;
 443         else
 444                 regval &= ~SEC_COMMON_CNT_CLR_CE_SNAP_EN;
 445         writel_relaxed(regval, addr);
 446 }
 447
 448 static void sec_ipv6_hashmask(struct sec_dev_info *info, u32 hash_mask[])
 449 {
 450         void __iomem *base = info->regs[SEC_SAA];
 451         int i;
 452
 453         for (i = 0; i < 10; i++)
 454                 writel_relaxed(hash_mask[0],
 455                                base + SEC_IPV6_MASK_TABLE_X_REG(i));
 456 }
 457
 458 static int sec_ipv4_hashmask(struct sec_dev_info *info, u32 hash_mask)
 459 {
 460         if (hash_mask & SEC_HASH_IPV4_MASK) {
 461                 dev_err(info->dev, "Sec Ipv4 Hash Mask Input Error!\n ");
 462                 return -EINVAL;
 463         }
 464
 465         writel_relaxed(hash_mask,
 466                        info->regs[SEC_SAA] + SEC_IPV4_MASK_TABLE_REG);
 467
 468         return 0;
 469 }
 470
 471 static void sec_set_dbg_bd_cfg(struct sec_dev_info *info, u32 cfg)
 472 {
 473         void __iomem *addr = info->regs[SEC_SAA] + SEC_DEBUG_BD_CFG_REG;
 474         u32 regval;
 475
 476         regval = readl_relaxed(addr);
 477         /* Always disable write back of normal bd */
 478         regval &= ~SEC_DEBUG_BD_CFG_WB_NORMAL;
 479
 480         if (cfg)
 481                 regval &= ~SEC_DEBUG_BD_CFG_WB_EN;
 482         else
 483                 regval |= SEC_DEBUG_BD_CFG_WB_EN;
 484
 485         writel_relaxed(regval, addr);
 486 }
 487
 488 static void sec_saa_getqm_en(struct sec_dev_info *info, u32 saa_indx, u32 en)
 489 {
 490         void __iomem *addr = info->regs[SEC_SAA] + SEC_SAA_BASE +
 491                 SEC_SAA_CTRL_REG(saa_indx);
 492         u32 regval;
 493
 494         regval = readl_relaxed(addr);
 495         if (en)
 496                 regval |= SEC_SAA_CTRL_GET_QM_EN;
 497         else
 498                 regval &= ~SEC_SAA_CTRL_GET_QM_EN;
 499         writel_relaxed(regval, addr);
 500 }
 501
 502 static void sec_saa_int_mask(struct sec_dev_info *info, u32 saa_indx,
 503                              u32 saa_int_mask)
 504 {
 505         writel_relaxed(saa_int_mask,
 506                        info->regs[SEC_SAA] + SEC_SAA_BASE + SEC_ST_INTMSK1_REG +
 507                        saa_indx * SEC_SAA_ADDR_SIZE);
 508 }
 509
 510 static void sec_streamid(struct sec_dev_info *info, int i)
 511 {
 512         #define SEC_SID 0x600
 513         #define SEC_VMID 0
 514
 515         writel_relaxed((SEC_VMID | ((SEC_SID & 0xffff) << 8)),
 516                        info->regs[SEC_SAA] + SEC_Q_VMID_CFG_REG(i));
 517 }
 518
 519 static void sec_queue_ar_alloc(struct sec_queue *queue, u32 alloc)
 520 {
 521         void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG;
 522         u32 regval;
 523
 524         regval = readl_relaxed(addr);
 525         if (alloc == SEC_QUEUE_AR_FROCE_ALLOC) {
 526                 regval |= SEC_Q_ARUSER_CFG_FA;
 527                 regval &= ~SEC_Q_ARUSER_CFG_FNA;
 528         } else {
 529                 regval &= ~SEC_Q_ARUSER_CFG_FA;
 530                 regval |= SEC_Q_ARUSER_CFG_FNA;
 531         }
 532
 533         writel_relaxed(regval, addr);
 534 }
 535
 536 static void sec_queue_aw_alloc(struct sec_queue *queue, u32 alloc)
 537 {
 538         void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
 539         u32 regval;
 540
 541         regval = readl_relaxed(addr);
 542         if (alloc == SEC_QUEUE_AW_FROCE_ALLOC) {
 543                 regval |= SEC_Q_AWUSER_CFG_FA;
 544                 regval &= ~SEC_Q_AWUSER_CFG_FNA;
 545         } else {
 546                 regval &= ~SEC_Q_AWUSER_CFG_FA;
 547                 regval |= SEC_Q_AWUSER_CFG_FNA;
 548         }
 549
 550         writel_relaxed(regval, addr);
 551 }
 552
 553 static void sec_queue_reorder(struct sec_queue *queue, bool reorder)
 554 {
 555         void __iomem *base = queue->regs;
 556         u32 regval;
 557
 558         regval = readl_relaxed(base + SEC_Q_CFG_REG);
 559         if (reorder)
 560                 regval |= SEC_Q_CFG_REORDER;
 561         else
 562                 regval &= ~SEC_Q_CFG_REORDER;
 563         writel_relaxed(regval, base + SEC_Q_CFG_REG);
 564 }
 565
 566 static void sec_queue_depth(struct sec_queue *queue, u32 depth)
 567 {
 568         void __iomem *addr = queue->regs + SEC_Q_DEPTH_CFG_REG;
 569         u32 regval;
 570
 571         regval = readl_relaxed(addr);
 572         regval &= ~SEC_Q_DEPTH_CFG_DEPTH_M;
 573         regval |= (depth << SEC_Q_DEPTH_CFG_DEPTH_S) & SEC_Q_DEPTH_CFG_DEPTH_M;
 574
 575         writel_relaxed(regval, addr);
 576 }
 577
 578 static void sec_queue_cmdbase_addr(struct sec_queue *queue, u64 addr)
 579 {
 580         writel_relaxed(upper_32_bits(addr), queue->regs + SEC_Q_BASE_HADDR_REG);
 581         writel_relaxed(lower_32_bits(addr), queue->regs + SEC_Q_BASE_LADDR_REG);
 582 }
 583
 584 static void sec_queue_outorder_addr(struct sec_queue *queue, u64 addr)
 585 {
 586         writel_relaxed(upper_32_bits(addr),
 587                        queue->regs + SEC_Q_OUTORDER_BASE_HADDR_REG);
 588         writel_relaxed(lower_32_bits(addr),
 589                        queue->regs + SEC_Q_OUTORDER_BASE_LADDR_REG);
 590 }
 591
 592 static void sec_queue_errbase_addr(struct sec_queue *queue, u64 addr)
 593 {
 594         writel_relaxed(upper_32_bits(addr),
 595                        queue->regs + SEC_Q_ERR_BASE_HADDR_REG);
 596         writel_relaxed(lower_32_bits(addr),
 597                        queue->regs + SEC_Q_ERR_BASE_LADDR_REG);
 598 }
 599
 600 static void sec_queue_irq_disable(struct sec_queue *queue)
 601 {
 602         writel_relaxed((u32)~0, queue->regs + SEC_Q_FLOW_INT_MKS_REG);
 603 }
 604
 605 static void sec_queue_irq_enable(struct sec_queue *queue)
 606 {
 607         writel_relaxed(0, queue->regs + SEC_Q_FLOW_INT_MKS_REG);
 608 }
 609
 610 static void sec_queue_abn_irq_disable(struct sec_queue *queue)
 611 {
 612         writel_relaxed((u32)~0, queue->regs + SEC_Q_FAIL_INT_MSK_REG);
 613 }
 614
 615 static void sec_queue_stop(struct sec_queue *queue)
 616 {
 617         disable_irq(queue->task_irq);
 618         sec_queue_irq_disable(queue);
 619         writel_relaxed(0x0, queue->regs + SEC_QUEUE_ENB_REG);
 620 }
 621
 622 static void sec_queue_start(struct sec_queue *queue)
 623 {
 624         sec_queue_irq_enable(queue);
 625         enable_irq(queue->task_irq);
 626         queue->expected = 0;
 627         writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG);
 628         writel_relaxed(0x1, queue->regs + SEC_QUEUE_ENB_REG);
 629 }
 630
 631 static struct sec_queue *sec_alloc_queue(struct sec_dev_info *info)
 632 {
 633         int i;
 634
 635         mutex_lock(&info->dev_lock);
 636
 637         /* Get the first idle queue in SEC device */
 638         for (i = 0; i < SEC_Q_NUM; i++)
 639                 if (!info->queues[i].in_use) {
 640                         info->queues[i].in_use = true;
 641                         info->queues_in_use++;
 642                         mutex_unlock(&info->dev_lock);
 643
 644                         return &info->queues[i];
 645                 }
 646         mutex_unlock(&info->dev_lock);
 647
 648         return ERR_PTR(-ENODEV);
 649 }
 650
 651 static int sec_queue_free(struct sec_queue *queue)
 652 {
 653         struct sec_dev_info *info = queue->dev_info;
 654
 655         if (queue->queue_id >= SEC_Q_NUM) {
 656                 dev_err(info->dev, "No queue %u\n", queue->queue_id);
 657                 return -ENODEV;
 658         }
 659
 660         if (!queue->in_use) {
 661                 dev_err(info->dev, "Queue %u is idle\n", queue->queue_id);
 662                 return -ENODEV;
 663         }
 664
 665         mutex_lock(&info->dev_lock);
 666         queue->in_use = false;
 667         info->queues_in_use--;
 668         mutex_unlock(&info->dev_lock);
 669
 670         return 0;
 671 }
 672
 673 static irqreturn_t sec_isr_handle_th(int irq, void *q)
 674 {
 675         sec_queue_irq_disable(q);
 676         return IRQ_WAKE_THREAD;
 677 }
 678
 679 static irqreturn_t sec_isr_handle(int irq, void *q)
 680 {
 681         struct sec_queue *queue = q;
 682         struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
 683         struct sec_queue_ring_cq *cq_ring = &queue->ring_cq;
 684         struct sec_out_bd_info *outorder_msg;
 685         struct sec_bd_info *msg;
 686         u32 ooo_read, ooo_write;
 687         void __iomem *base = queue->regs;
 688         int q_id;
 689
 690         ooo_read = readl(base + SEC_Q_OUTORDER_RD_PTR_REG);
 691         ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
 692         outorder_msg = cq_ring->vaddr + ooo_read;
 693         q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
 694         msg = msg_ring->vaddr + q_id;
 695
 696         while ((ooo_write != ooo_read) && msg->w0 & SEC_BD_W0_DONE) {
 697                 /*
 698                  * Must be before callback otherwise blocks adding other chained
 699                  * elements
 700                  */
 701                 set_bit(q_id, queue->unprocessed);
 702                 if (q_id == queue->expected)
 703                         while (test_bit(queue->expected, queue->unprocessed)) {
 704                                 clear_bit(queue->expected, queue->unprocessed);
 705                                 msg = msg_ring->vaddr + queue->expected;
 706                                 msg->w0 &= ~SEC_BD_W0_DONE;
 707                                 msg_ring->callback(msg,
 708                                                 queue->shadow[queue->expected]);
 709                                 queue->shadow[queue->expected] = NULL;
 710                                 queue->expected = (queue->expected + 1) %
 711                                         SEC_QUEUE_LEN;
 712                                 atomic_dec(&msg_ring->used);
 713                         }
 714
 715                 ooo_read = (ooo_read + 1) % SEC_QUEUE_LEN;
 716                 writel(ooo_read, base + SEC_Q_OUTORDER_RD_PTR_REG);
 717                 ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
 718                 outorder_msg = cq_ring->vaddr + ooo_read;
 719                 q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
 720                 msg = msg_ring->vaddr + q_id;
 721         }
 722
 723         sec_queue_irq_enable(queue);
 724
 725         return IRQ_HANDLED;
 726 }
 727
 728 static int sec_queue_irq_init(struct sec_queue *queue)
 729 {
 730         struct sec_dev_info *info = queue->dev_info;
 731         int irq = queue->task_irq;
 732         int ret;
 733
 734         ret = request_threaded_irq(irq, sec_isr_handle_th, sec_isr_handle,
 735                                    IRQF_TRIGGER_RISING, queue->name, queue);
 736         if (ret) {
 737                 dev_err(info->dev, "request irq(%d) failed %d\n", irq, ret);
 738                 return ret;
 739         }
 740         disable_irq(irq);
 741
 742         return 0;
 743 }
 744
 745 static int sec_queue_irq_uninit(struct sec_queue *queue)
 746 {
 747         free_irq(queue->task_irq, queue);
 748
 749         return 0;
 750 }
 751
 752 static struct sec_dev_info *sec_device_get(void)
 753 {
 754         struct sec_dev_info *sec_dev = NULL;
 755         struct sec_dev_info *this_sec_dev;
 756         int least_busy_n = SEC_Q_NUM + 1;
 757         int i;
 758
 759         /* Find which one is least busy and use that first */
 760         for (i = 0; i < SEC_MAX_DEVICES; i++) {
 761                 this_sec_dev = sec_devices[i];
 762                 if (this_sec_dev &&
 763                     this_sec_dev->queues_in_use < least_busy_n) {
 764                         least_busy_n = this_sec_dev->queues_in_use;
 765                         sec_dev = this_sec_dev;
 766                 }
 767         }
 768
 769         return sec_dev;
 770 }
 771
 772 static struct sec_queue *sec_queue_alloc_start(struct sec_dev_info *info)
 773 {
 774         struct sec_queue *queue;
 775
 776         queue = sec_alloc_queue(info);
 777         if (IS_ERR(queue)) {
 778                 dev_err(info->dev, "alloc sec queue failed! %ld\n",
 779                         PTR_ERR(queue));
 780                 return queue;
 781         }
 782
 783         sec_queue_start(queue);
 784
 785         return queue;
 786 }
 787
 788 /**
 789  * sec_queue_alloc_start_safe - get a hw queue from appropriate instance
 790  *
 791  * This function does extremely simplistic load balancing. It does not take into
 792  * account NUMA locality of the accelerator, or which cpu has requested the
 793  * queue.  Future work may focus on optimizing this in order to improve full
 794  * machine throughput.
 795  */
 796 struct sec_queue *sec_queue_alloc_start_safe(void)
 797 {
 798         struct sec_dev_info *info;
 799         struct sec_queue *queue = ERR_PTR(-ENODEV);
 800
 801         mutex_lock(&sec_id_lock);
 802         info = sec_device_get();
 803         if (!info)
 804                 goto unlock;
 805
 806         queue = sec_queue_alloc_start(info);
 807
 808 unlock:
 809         mutex_unlock(&sec_id_lock);
 810
 811         return queue;
 812 }
 813
 814 /**
 815  * sec_queue_stop_release() - free up a hw queue for reuse
 816  * @queue: The queue we are done with.
 817  *
 818  * This will stop the current queue, terminanting any transactions
 819  * that are inflight an return it to the pool of available hw queuess
 820  */
 821 int sec_queue_stop_release(struct sec_queue *queue)
 822 {
 823         struct device *dev = queue->dev_info->dev;
 824         int ret;
 825
 826         sec_queue_stop(queue);
 827
 828         ret = sec_queue_free(queue);
 829         if (ret)
 830                 dev_err(dev, "Releasing queue failed %d\n", ret);
 831
 832         return ret;
 833 }
 834
 835 /**
 836  * sec_queue_empty() - Is this hardware queue currently empty.
 837  * @queue: The queue to test
 838  *
 839  * We need to know if we have an empty queue for some of the chaining modes
 840  * as if it is not empty we may need to hold the message in a software queue
 841  * until the hw queue is drained.
 842  */
 843 bool sec_queue_empty(struct sec_queue *queue)
 844 {
 845         struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
 846
 847         return !atomic_read(&msg_ring->used);
 848 }
 849
 850 /**
 851  * sec_queue_send() - queue up a single operation in the hw queue
 852  * @queue: The queue in which to put the message
 853  * @msg: The message
 854  * @ctx: Context to be put in the shadow array and passed back to cb on result.
 855  *
 856  * This function will return -EAGAIN if the queue is currently full.
 857  */
 858 int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx)
 859 {
 860         struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
 861         void __iomem *base = queue->regs;
 862         u32 write, read;
 863
 864         mutex_lock(&msg_ring->lock);
 865         read = readl(base + SEC_Q_RD_PTR_REG);
 866         write = readl(base + SEC_Q_WR_PTR_REG);
 867         if (write == read && atomic_read(&msg_ring->used) == SEC_QUEUE_LEN) {
 868                 mutex_unlock(&msg_ring->lock);
 869                 return -EAGAIN;
 870         }
 871         memcpy(msg_ring->vaddr + write, msg, sizeof(*msg));
 872         queue->shadow[write] = ctx;
 873         write = (write + 1) % SEC_QUEUE_LEN;
 874
 875         /* Ensure content updated before queue advance */
 876         wmb();
 877         writel(write, base + SEC_Q_WR_PTR_REG);
 878
 879         atomic_inc(&msg_ring->used);
 880         mutex_unlock(&msg_ring->lock);
 881
 882         return 0;
 883 }
 884
 885 bool sec_queue_can_enqueue(struct sec_queue *queue, int num)
 886 {
 887         struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
 888
 889         return SEC_QUEUE_LEN - atomic_read(&msg_ring->used) >= num;
 890 }
 891
 892 static void sec_queue_hw_init(struct sec_queue *queue)
 893 {
 894         sec_queue_ar_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC);
 895         sec_queue_aw_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC);
 896         sec_queue_ar_pkgattr(queue, 1);
 897         sec_queue_aw_pkgattr(queue, 1);
 898
 899         /* Enable out of order queue */
 900         sec_queue_reorder(queue, true);
 901
 902         /* Interrupt after a single complete element */
 903         writel_relaxed(1, queue->regs + SEC_Q_PROC_NUM_CFG_REG);
 904
 905         sec_queue_depth(queue, SEC_QUEUE_LEN - 1);
 906
 907         sec_queue_cmdbase_addr(queue, queue->ring_cmd.paddr);
 908
 909         sec_queue_outorder_addr(queue, queue->ring_cq.paddr);
 910
 911         sec_queue_errbase_addr(queue, queue->ring_db.paddr);
 912
 913         writel_relaxed(0x100, queue->regs + SEC_Q_OT_TH_REG);
 914
 915         sec_queue_abn_irq_disable(queue);
 916         sec_queue_irq_disable(queue);
 917         writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG);
 918 }
 919
 920 static int sec_hw_init(struct sec_dev_info *info)
 921 {
 922         struct iommu_domain *domain;
 923         u32 sec_ipv4_mask = 0;
 924         u32 sec_ipv6_mask[10] = {};
 925         u32 i, ret;
 926
 927         domain = iommu_get_domain_for_dev(info->dev);
 928
 929         /*
 930          * Enable all available processing unit clocks.
 931          * Only the first cluster is usable with translations.
 932          */
 933         if (domain && (domain->type & __IOMMU_DOMAIN_PAGING))
 934                 info->num_saas = 5;
 935
 936         else
 937                 info->num_saas = 10;
 938
 939         writel_relaxed(GENMASK(info->num_saas - 1, 0),
 940                        info->regs[SEC_SAA] + SEC_CLK_EN_REG);
 941
 942         /* 32 bit little endian */
 943         sec_bd_endian_little(info);
 944
 945         sec_cache_config(info);
 946
 947         /* Data axi port write and read outstanding config as per datasheet */
 948         sec_data_axiwr_otsd_cfg(info, 0x7);
 949         sec_data_axird_otsd_cfg(info, 0x7);
 950
 951         /* Enable clock gating */
 952         sec_clk_gate_en(info, true);
 953
 954         /* Set CNT_CYC register not read clear */
 955         sec_comm_cnt_cfg(info, false);
 956
 957         /* Enable CNT_CYC */
 958         sec_commsnap_en(info, false);
 959
 960         writel_relaxed((u32)~0, info->regs[SEC_SAA] + SEC_FSM_MAX_CNT_REG);
 961
 962         ret = sec_ipv4_hashmask(info, sec_ipv4_mask);
 963         if (ret) {
 964                 dev_err(info->dev, "Failed to set ipv4 hashmask %d\n", ret);
 965                 return -EIO;
 966         }
 967
 968         sec_ipv6_hashmask(info, sec_ipv6_mask);
 969
 970         /*  do not use debug bd */
 971         sec_set_dbg_bd_cfg(info, 0);
 972
 973         if (domain && (domain->type & __IOMMU_DOMAIN_PAGING)) {
 974                 for (i = 0; i < SEC_Q_NUM; i++) {
 975                         sec_streamid(info, i);
 976                         /* Same QoS for all queues */
 977                         writel_relaxed(0x3f,
 978                                        info->regs[SEC_SAA] +
 979                                        SEC_Q_WEIGHT_CFG_REG(i));
 980                 }
 981         }
 982
 983         for (i = 0; i < info->num_saas; i++) {
 984                 sec_saa_getqm_en(info, i, 1);
 985                 sec_saa_int_mask(info, i, 0);
 986         }
 987
 988         return 0;
 989 }
 990
 991 static void sec_hw_exit(struct sec_dev_info *info)
 992 {
 993         int i;
 994
 995         for (i = 0; i < SEC_MAX_SAA_NUM; i++) {
 996                 sec_saa_int_mask(info, i, (u32)~0);
 997                 sec_saa_getqm_en(info, i, 0);
 998         }
 999 }
1000
1001 static void sec_queue_base_init(struct sec_dev_info *info,
1002                                 struct sec_queue *queue, int queue_id)
1003 {
1004         queue->dev_info = info;
1005         queue->queue_id = queue_id;
1006         snprintf(queue->name, sizeof(queue->name),
1007                  "%s_%d", dev_name(info->dev), queue->queue_id);
1008 }
1009
1010 static int sec_map_io(struct sec_dev_info *info, struct platform_device *pdev)
1011 {
1012         struct resource *res;
1013         int i;
1014
1015         for (i = 0; i < SEC_NUM_ADDR_REGIONS; i++) {
1016                 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
1017
1018                 if (!res) {
1019                         dev_err(info->dev, "Memory resource %d not found\n", i);
1020                         return -EINVAL;
1021                 }
1022
1023                 info->regs[i] = devm_ioremap(info->dev, res->start,
1024                                              resource_size(res));
1025                 if (!info->regs[i]) {
1026                         dev_err(info->dev,
1027                                 "Memory resource %d could not be remapped\n",
1028                                 i);
1029                         return -EINVAL;
1030                 }
1031         }
1032
1033         return 0;
1034 }
1035
1036 static int sec_base_init(struct sec_dev_info *info,
1037                          struct platform_device *pdev)
1038 {
1039         int ret;
1040
1041         ret = sec_map_io(info, pdev);
1042         if (ret)
1043                 return ret;
1044
1045         ret = sec_clk_en(info);
1046         if (ret)
1047                 return ret;
1048
1049         ret = sec_reset_whole_module(info);
1050         if (ret)
1051                 goto sec_clk_disable;
1052
1053         ret = sec_hw_init(info);
1054         if (ret)
1055                 goto sec_clk_disable;
1056
1057         return 0;
1058
1059 sec_clk_disable:
1060         sec_clk_dis(info);
1061
1062         return ret;
1063 }
1064
1065 static void sec_base_exit(struct sec_dev_info *info)
1066 {
1067         sec_hw_exit(info);
1068         sec_clk_dis(info);
1069 }
1070
1071 #define SEC_Q_CMD_SIZE \
1072         round_up(SEC_QUEUE_LEN * sizeof(struct sec_bd_info), PAGE_SIZE)
1073 #define SEC_Q_CQ_SIZE \
1074         round_up(SEC_QUEUE_LEN * sizeof(struct sec_out_bd_info), PAGE_SIZE)
1075 #define SEC_Q_DB_SIZE \
1076         round_up(SEC_QUEUE_LEN * sizeof(struct sec_debug_bd_info), PAGE_SIZE)
1077
1078 static int sec_queue_res_cfg(struct sec_queue *queue)
1079 {
1080         struct device *dev = queue->dev_info->dev;
1081         struct sec_queue_ring_cmd *ring_cmd = &queue->ring_cmd;
1082         struct sec_queue_ring_cq *ring_cq = &queue->ring_cq;
1083         struct sec_queue_ring_db *ring_db = &queue->ring_db;
1084         int ret;
1085
1086         ring_cmd->vaddr = dma_alloc_coherent(dev, SEC_Q_CMD_SIZE,
1087                                              &ring_cmd->paddr, GFP_KERNEL);
1088         if (!ring_cmd->vaddr)
1089                 return -ENOMEM;
1090
1091         atomic_set(&ring_cmd->used, 0);
1092         mutex_init(&ring_cmd->lock);
1093         ring_cmd->callback = sec_alg_callback;
1094
1095         ring_cq->vaddr = dma_alloc_coherent(dev, SEC_Q_CQ_SIZE,
1096                                             &ring_cq->paddr, GFP_KERNEL);
1097         if (!ring_cq->vaddr) {
1098                 ret = -ENOMEM;
1099                 goto err_free_ring_cmd;
1100         }
1101
1102         ring_db->vaddr = dma_alloc_coherent(dev, SEC_Q_DB_SIZE,
1103                                             &ring_db->paddr, GFP_KERNEL);
1104         if (!ring_db->vaddr) {
1105                 ret = -ENOMEM;
1106                 goto err_free_ring_cq;
1107         }
1108         queue->task_irq = platform_get_irq(to_platform_device(dev),
1109                                            queue->queue_id * 2 + 1);
1110         if (queue->task_irq <= 0) {
1111                 ret = -EINVAL;
1112                 goto err_free_ring_db;
1113         }
1114
1115         return 0;
1116
1117 err_free_ring_db:
1118         dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr,
1119                           queue->ring_db.paddr);
1120 err_free_ring_cq:
1121         dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr,
1122                           queue->ring_cq.paddr);
1123 err_free_ring_cmd:
1124         dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr,
1125                           queue->ring_cmd.paddr);
1126
1127         return ret;
1128 }
1129
1130 static void sec_queue_free_ring_pages(struct sec_queue *queue)
1131 {
1132         struct device *dev = queue->dev_info->dev;
1133
1134         dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr,
1135                           queue->ring_db.paddr);
1136         dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr,
1137                           queue->ring_cq.paddr);
1138         dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr,
1139                           queue->ring_cmd.paddr);
1140 }
1141
1142 static int sec_queue_config(struct sec_dev_info *info, struct sec_queue *queue,
1143                             int queue_id)
1144 {
1145         int ret;
1146
1147         sec_queue_base_init(info, queue, queue_id);
1148
1149         ret = sec_queue_res_cfg(queue);
1150         if (ret)
1151                 return ret;
1152
1153         ret = sec_queue_map_io(queue);
1154         if (ret) {
1155                 dev_err(info->dev, "Queue map failed %d\n", ret);
1156                 sec_queue_free_ring_pages(queue);
1157                 return ret;
1158         }
1159
1160         sec_queue_hw_init(queue);
1161
1162         return 0;
1163 }
1164
1165 static void sec_queue_unconfig(struct sec_dev_info *info,
1166                                struct sec_queue *queue)
1167 {
1168         sec_queue_unmap_io(queue);
1169         sec_queue_free_ring_pages(queue);
1170 }
1171
1172 static int sec_id_alloc(struct sec_dev_info *info)
1173 {
1174         int ret = 0;
1175         int i;
1176
1177         mutex_lock(&sec_id_lock);
1178
1179         for (i = 0; i < SEC_MAX_DEVICES; i++)
1180                 if (!sec_devices[i])
1181                         break;
1182         if (i == SEC_MAX_DEVICES) {
1183                 ret = -ENOMEM;
1184                 goto unlock;
1185         }
1186         info->sec_id = i;
1187         sec_devices[info->sec_id] = info;
1188
1189 unlock:
1190         mutex_unlock(&sec_id_lock);
1191
1192         return ret;
1193 }
1194
1195 static void sec_id_free(struct sec_dev_info *info)
1196 {
1197         mutex_lock(&sec_id_lock);
1198         sec_devices[info->sec_id] = NULL;
1199         mutex_unlock(&sec_id_lock);
1200 }
1201
1202 static int sec_probe(struct platform_device *pdev)
1203 {
1204         struct sec_dev_info *info;
1205         struct device *dev = &pdev->dev;
1206         int i, j;
1207         int ret;
1208
1209         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1210         if (ret) {
1211                 dev_err(dev, "Failed to set 64 bit dma mask %d", ret);
1212                 return -ENODEV;
1213         }
1214
1215         info = devm_kzalloc(dev, (sizeof(*info)), GFP_KERNEL);
1216         if (!info)
1217                 return -ENOMEM;
1218
1219         info->dev = dev;
1220         mutex_init(&info->dev_lock);
1221
1222         info->hw_sgl_pool = dmam_pool_create("sgl", dev,
1223                                              sizeof(struct sec_hw_sgl), 64, 0);
1224         if (!info->hw_sgl_pool) {
1225                 dev_err(dev, "Failed to create sec sgl dma pool\n");
1226                 return -ENOMEM;
1227         }
1228
1229         ret = sec_base_init(info, pdev);
1230         if (ret) {
1231                 dev_err(dev, "Base initialization fail! %d\n", ret);
1232                 return ret;
1233         }
1234
1235         for (i = 0; i < SEC_Q_NUM; i++) {
1236                 ret = sec_queue_config(info, &info->queues[i], i);
1237                 if (ret)
1238                         goto queues_unconfig;
1239
1240                 ret = sec_queue_irq_init(&info->queues[i]);
1241                 if (ret) {
1242                         sec_queue_unconfig(info, &info->queues[i]);
1243                         goto queues_unconfig;
1244                 }
1245         }
1246
1247         ret = sec_algs_register();
1248         if (ret) {
1249                 dev_err(dev, "Failed to register algorithms with crypto %d\n",
1250                         ret);
1251                 goto queues_unconfig;
1252         }
1253
1254         platform_set_drvdata(pdev, info);
1255
1256         ret = sec_id_alloc(info);
1257         if (ret)
1258                 goto algs_unregister;
1259
1260         return 0;
1261
1262 algs_unregister:
1263         sec_algs_unregister();
1264 queues_unconfig:
1265         for (j = i - 1; j >= 0; j--) {
1266                 sec_queue_irq_uninit(&info->queues[j]);
1267                 sec_queue_unconfig(info, &info->queues[j]);
1268         }
1269         sec_base_exit(info);
1270
1271         return ret;
1272 }
1273
1274 static int sec_remove(struct platform_device *pdev)
1275 {
1276         struct sec_dev_info *info = platform_get_drvdata(pdev);
1277         int i;
1278
1279         /* Unexpose as soon as possible, reuse during remove is fine */
1280         sec_id_free(info);
1281
1282         sec_algs_unregister();
1283
1284         for (i = 0; i < SEC_Q_NUM; i++) {
1285                 sec_queue_irq_uninit(&info->queues[i]);
1286                 sec_queue_unconfig(info, &info->queues[i]);
1287         }
1288
1289         sec_base_exit(info);
1290
1291         return 0;
1292 }
1293
1294 static const __maybe_unused struct of_device_id sec_match[] = {
1295         { .compatible = "hisilicon,hip06-sec" },
1296         { .compatible = "hisilicon,hip07-sec" },
1297         {}
1298 };
1299 MODULE_DEVICE_TABLE(of, sec_match);
1300
1301 static const __maybe_unused struct acpi_device_id sec_acpi_match[] = {
1302         { "HISI02C1", 0 },
1303         { }
1304 };
1305 MODULE_DEVICE_TABLE(acpi, sec_acpi_match);
1306
1307 static struct platform_driver sec_driver = {
1308         .probe = sec_probe,
1309         .remove = sec_remove,
1310         .driver = {
1311                 .name = "hisi_sec_platform_driver",
1312                 .of_match_table = sec_match,
1313                 .acpi_match_table = ACPI_PTR(sec_acpi_match),
1314         },
1315 };
1316 module_platform_driver(sec_driver);
1317
1318 MODULE_LICENSE("GPL");
1319 MODULE_DESCRIPTION("HiSilicon Security Accelerators");
1320 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com");
1321 MODULE_AUTHOR("Jonathan Cameron <jonathan.cameron@huawei.com>");