drivers/crypto/qat/qat_common/qat_hal.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/slab.h>
   4 #include <linux/delay.h>
   5 #include <linux/pci_ids.h>
   6
   7 #include "adf_accel_devices.h"
   8 #include "adf_common_drv.h"
   9 #include "icp_qat_hal.h"
  10 #include "icp_qat_uclo.h"
  11
  12 #define BAD_REGADDR            0xffff
  13 #define MAX_RETRY_TIMES    10000
  14 #define INIT_CTX_ARB_VALUE      0x0
  15 #define INIT_CTX_ENABLE_VALUE     0x0
  16 #define INIT_PC_VALUE        0x0
  17 #define INIT_WAKEUP_EVENTS_VALUE  0x1
  18 #define INIT_SIG_EVENTS_VALUE     0x1
  19 #define INIT_CCENABLE_VALUE       0x2000
  20 #define RST_CSR_QAT_LSB    20
  21 #define RST_CSR_AE_LSB            0
  22 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  23
  24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  25         (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  26         (~(1 << CE_REG_PAR_ERR_BITPOS)))
  27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  28         (inst = ((inst & 0xFFFF00C03FFull) | \
  29                 ((((const_val) << 12) & 0x0FF00000ull) | \
  30                 (((const_val) << 10) & 0x0003FC00ull))))
  31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  32         (inst = ((inst & 0xFFFF00FFF00ull) | \
  33                 ((((const_val) << 12) & 0x0FF00000ull) | \
  34                 (((const_val) <<  0) & 0x000000FFull))))
  35
  36 #define AE(handle, ae) ((handle)->hal_handle->aes[ae])
  37
  38 static const u64 inst_4b[] = {
  39         0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  40         0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  41         0x0A021000000ull
  42 };
  43
  44 static const u64 inst[] = {
  45         0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  46         0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  47         0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  48         0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  49         0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  50         0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  51         0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  52         0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  53         0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  54         0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  55         0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  56         0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  57         0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
  58         0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
  59         0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
  60         0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
  61         0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
  62         0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
  63         0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
  64         0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
  65         0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
  66         0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
  67 };
  68
  69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
  70                           unsigned char ae, unsigned int ctx_mask)
  71 {
  72         AE(handle, ae).live_ctx_mask = ctx_mask;
  73 }
  74
  75 #define CSR_RETRY_TIMES 500
  76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
  77                              unsigned char ae, unsigned int csr)
  78 {
  79         unsigned int iterations = CSR_RETRY_TIMES;
  80         int value;
  81
  82         do {
  83                 value = GET_AE_CSR(handle, ae, csr);
  84                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
  85                         return value;
  86         } while (iterations--);
  87
  88         pr_err("QAT: Read CSR timeout\n");
  89         return 0;
  90 }
  91
  92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
  93                              unsigned char ae, unsigned int csr,
  94                              unsigned int value)
  95 {
  96         unsigned int iterations = CSR_RETRY_TIMES;
  97
  98         do {
  99                 SET_AE_CSR(handle, ae, csr, value);
 100                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 101                         return 0;
 102         } while (iterations--);
 103
 104         pr_err("QAT: Write CSR Timeout\n");
 105         return -EFAULT;
 106 }
 107
 108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 109                                      unsigned char ae, unsigned char ctx,
 110                                      unsigned int *events)
 111 {
 112         unsigned int cur_ctx;
 113
 114         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 115         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 116         *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
 117         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 118 }
 119
 120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 121                                unsigned char ae, unsigned int cycles,
 122                                int chk_inactive)
 123 {
 124         unsigned int base_cnt = 0, cur_cnt = 0;
 125         unsigned int csr = (1 << ACS_ABO_BITPOS);
 126         int times = MAX_RETRY_TIMES;
 127         int elapsed_cycles = 0;
 128
 129         base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 130         base_cnt &= 0xffff;
 131         while ((int)cycles > elapsed_cycles && times--) {
 132                 if (chk_inactive)
 133                         csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 134
 135                 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 136                 cur_cnt &= 0xffff;
 137                 elapsed_cycles = cur_cnt - base_cnt;
 138
 139                 if (elapsed_cycles < 0)
 140                         elapsed_cycles += 0x10000;
 141
 142                 /* ensure at least 8 time cycles elapsed in wait_cycles */
 143                 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 144                         return 0;
 145         }
 146         if (times < 0) {
 147                 pr_err("QAT: wait_num_cycles time out\n");
 148                 return -EFAULT;
 149         }
 150         return 0;
 151 }
 152
 153 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
 154 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
 155
 156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 157                             unsigned char ae, unsigned char mode)
 158 {
 159         unsigned int csr, new_csr;
 160
 161         if (mode != 4 && mode != 8) {
 162                 pr_err("QAT: bad ctx mode=%d\n", mode);
 163                 return -EINVAL;
 164         }
 165
 166         /* Sets the accelaration engine context mode to either four or eight */
 167         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 168         csr = IGNORE_W1C_MASK & csr;
 169         new_csr = (mode == 4) ?
 170                 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 171                 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 172         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 173         return 0;
 174 }
 175
 176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 177                            unsigned char ae, unsigned char mode)
 178 {
 179         unsigned int csr, new_csr;
 180
 181         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 182         csr &= IGNORE_W1C_MASK;
 183
 184         new_csr = (mode) ?
 185                 SET_BIT(csr, CE_NN_MODE_BITPOS) :
 186                 CLR_BIT(csr, CE_NN_MODE_BITPOS);
 187
 188         if (new_csr != csr)
 189                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 190
 191         return 0;
 192 }
 193
 194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 195                            unsigned char ae, enum icp_qat_uof_regtype lm_type,
 196                            unsigned char mode)
 197 {
 198         unsigned int csr, new_csr;
 199
 200         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 201         csr &= IGNORE_W1C_MASK;
 202         switch (lm_type) {
 203         case ICP_LMEM0:
 204                 new_csr = (mode) ?
 205                         SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 206                         CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 207                 break;
 208         case ICP_LMEM1:
 209                 new_csr = (mode) ?
 210                         SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 211                         CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 212                 break;
 213         case ICP_LMEM2:
 214                 new_csr = (mode) ?
 215                         SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
 216                         CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
 217                 break;
 218         case ICP_LMEM3:
 219                 new_csr = (mode) ?
 220                         SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
 221                         CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
 222                 break;
 223         default:
 224                 pr_err("QAT: lmType = 0x%x\n", lm_type);
 225                 return -EINVAL;
 226         }
 227
 228         if (new_csr != csr)
 229                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 230         return 0;
 231 }
 232
 233 void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
 234                                 unsigned char ae, unsigned char mode)
 235 {
 236         unsigned int csr, new_csr;
 237
 238         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 239         csr &= IGNORE_W1C_MASK;
 240         new_csr = (mode) ?
 241                   SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
 242                   CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
 243         if (new_csr != csr)
 244                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 245 }
 246
 247 static unsigned short qat_hal_get_reg_addr(unsigned int type,
 248                                            unsigned short reg_num)
 249 {
 250         unsigned short reg_addr;
 251
 252         switch (type) {
 253         case ICP_GPA_ABS:
 254         case ICP_GPB_ABS:
 255                 reg_addr = 0x80 | (reg_num & 0x7f);
 256                 break;
 257         case ICP_GPA_REL:
 258         case ICP_GPB_REL:
 259                 reg_addr = reg_num & 0x1f;
 260                 break;
 261         case ICP_SR_RD_REL:
 262         case ICP_SR_WR_REL:
 263         case ICP_SR_REL:
 264                 reg_addr = 0x180 | (reg_num & 0x1f);
 265                 break;
 266         case ICP_SR_ABS:
 267                 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 268                 break;
 269         case ICP_DR_RD_REL:
 270         case ICP_DR_WR_REL:
 271         case ICP_DR_REL:
 272                 reg_addr = 0x1c0 | (reg_num & 0x1f);
 273                 break;
 274         case ICP_DR_ABS:
 275                 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 276                 break;
 277         case ICP_NEIGH_REL:
 278                 reg_addr = 0x280 | (reg_num & 0x1f);
 279                 break;
 280         case ICP_LMEM0:
 281                 reg_addr = 0x200;
 282                 break;
 283         case ICP_LMEM1:
 284                 reg_addr = 0x220;
 285                 break;
 286         case ICP_LMEM2:
 287                 reg_addr = 0x2c0;
 288                 break;
 289         case ICP_LMEM3:
 290                 reg_addr = 0x2e0;
 291                 break;
 292         case ICP_NO_DEST:
 293                 reg_addr = 0x300 | (reg_num & 0xff);
 294                 break;
 295         default:
 296                 reg_addr = BAD_REGADDR;
 297                 break;
 298         }
 299         return reg_addr;
 300 }
 301
 302 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 303 {
 304         unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 305         unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 306         unsigned int csr_val;
 307
 308         csr_val = GET_CAP_CSR(handle, reset_csr);
 309         csr_val |= reset_mask;
 310         SET_CAP_CSR(handle, reset_csr, csr_val);
 311 }
 312
 313 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 314                                 unsigned char ae, unsigned int ctx_mask,
 315                                 unsigned int ae_csr, unsigned int csr_val)
 316 {
 317         unsigned int ctx, cur_ctx;
 318
 319         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 320
 321         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 322                 if (!(ctx_mask & (1 << ctx)))
 323                         continue;
 324                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 325                 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 326         }
 327
 328         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 329 }
 330
 331 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 332                                 unsigned char ae, unsigned char ctx,
 333                                 unsigned int ae_csr)
 334 {
 335         unsigned int cur_ctx, csr_val;
 336
 337         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 338         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 339         csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
 340         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 341
 342         return csr_val;
 343 }
 344
 345 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 346                                   unsigned char ae, unsigned int ctx_mask,
 347                                   unsigned int events)
 348 {
 349         unsigned int ctx, cur_ctx;
 350
 351         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 352         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 353                 if (!(ctx_mask & (1 << ctx)))
 354                         continue;
 355                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 356                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 357         }
 358         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 359 }
 360
 361 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 362                                      unsigned char ae, unsigned int ctx_mask,
 363                                      unsigned int events)
 364 {
 365         unsigned int ctx, cur_ctx;
 366
 367         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 368         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 369                 if (!(ctx_mask & (1 << ctx)))
 370                         continue;
 371                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 372                 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 373                                   events);
 374         }
 375         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 376 }
 377
 378 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 379 {
 380         unsigned long ae_mask = handle->hal_handle->ae_mask;
 381         unsigned int base_cnt, cur_cnt;
 382         unsigned char ae;
 383         int times = MAX_RETRY_TIMES;
 384
 385         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 386                 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 387                 base_cnt &= 0xffff;
 388
 389                 do {
 390                         cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 391                         cur_cnt &= 0xffff;
 392                 } while (times-- && (cur_cnt == base_cnt));
 393
 394                 if (times < 0) {
 395                         pr_err("QAT: AE%d is inactive!!\n", ae);
 396                         return -EFAULT;
 397                 }
 398         }
 399
 400         return 0;
 401 }
 402
 403 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
 404                             unsigned int ae)
 405 {
 406         unsigned int enable = 0, active = 0;
 407
 408         enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 409         active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 410         if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
 411             (active & (1 << ACS_ABO_BITPOS)))
 412                 return 1;
 413         else
 414                 return 0;
 415 }
 416
 417 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 418 {
 419         unsigned long ae_mask = handle->hal_handle->ae_mask;
 420         unsigned int misc_ctl_csr, misc_ctl;
 421         unsigned char ae;
 422
 423         misc_ctl_csr = handle->chip_info->misc_ctl_csr;
 424         /* stop the timestamp timers */
 425         misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);
 426         if (misc_ctl & MC_TIMESTAMP_ENABLE)
 427                 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &
 428                             (~MC_TIMESTAMP_ENABLE));
 429
 430         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 431                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 432                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 433         }
 434         /* start timestamp timers */
 435         SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
 436 }
 437
 438 #define ESRAM_AUTO_TINIT        BIT(2)
 439 #define ESRAM_AUTO_TINIT_DONE   BIT(3)
 440 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 441 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 442 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 443 {
 444         void __iomem *csr_addr =
 445                         (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
 446                         ESRAM_AUTO_INIT_CSR_OFFSET);
 447         unsigned int csr_val;
 448         int times = 30;
 449
 450         if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
 451                 return 0;
 452
 453         csr_val = ADF_CSR_RD(csr_addr, 0);
 454         if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 455                 return 0;
 456
 457         csr_val = ADF_CSR_RD(csr_addr, 0);
 458         csr_val |= ESRAM_AUTO_TINIT;
 459         ADF_CSR_WR(csr_addr, 0, csr_val);
 460
 461         do {
 462                 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 463                 csr_val = ADF_CSR_RD(csr_addr, 0);
 464         } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 465         if (times < 0) {
 466                 pr_err("QAT: Fail to init eSram!\n");
 467                 return -EFAULT;
 468         }
 469         return 0;
 470 }
 471
 472 #define SHRAM_INIT_CYCLES 2060
 473 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 474 {
 475         unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;
 476         unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 477         unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 478         unsigned long ae_mask = handle->hal_handle->ae_mask;
 479         unsigned char ae = 0;
 480         unsigned int times = 100;
 481         unsigned int csr_val;
 482
 483         /* write to the reset csr */
 484         csr_val = GET_CAP_CSR(handle, reset_csr);
 485         csr_val &= ~reset_mask;
 486         do {
 487                 SET_CAP_CSR(handle, reset_csr, csr_val);
 488                 if (!(times--))
 489                         goto out_err;
 490                 csr_val = GET_CAP_CSR(handle, reset_csr);
 491                 csr_val &= reset_mask;
 492         } while (csr_val);
 493         /* enable clock */
 494         csr_val = GET_CAP_CSR(handle, clk_csr);
 495         csr_val |= reset_mask;
 496         SET_CAP_CSR(handle, clk_csr, csr_val);
 497         if (qat_hal_check_ae_alive(handle))
 498                 goto out_err;
 499
 500         /* Set undefined power-up/reset states to reasonable default values */
 501         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 502                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 503                                   INIT_CTX_ENABLE_VALUE);
 504                 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 505                                     CTX_STS_INDIRECT,
 506                                     handle->hal_handle->upc_mask &
 507                                     INIT_PC_VALUE);
 508                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 509                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 510                 qat_hal_put_wakeup_event(handle, ae,
 511                                          ICP_QAT_UCLO_AE_ALL_CTX,
 512                                          INIT_WAKEUP_EVENTS_VALUE);
 513                 qat_hal_put_sig_event(handle, ae,
 514                                       ICP_QAT_UCLO_AE_ALL_CTX,
 515                                       INIT_SIG_EVENTS_VALUE);
 516         }
 517         if (qat_hal_init_esram(handle))
 518                 goto out_err;
 519         if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 520                 goto out_err;
 521         qat_hal_reset_timestamp(handle);
 522
 523         return 0;
 524 out_err:
 525         pr_err("QAT: failed to get device out of reset\n");
 526         return -EFAULT;
 527 }
 528
 529 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 530                                 unsigned char ae, unsigned int ctx_mask)
 531 {
 532         unsigned int ctx;
 533
 534         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 535         ctx &= IGNORE_W1C_MASK &
 536                 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 537         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 538 }
 539
 540 static u64 qat_hal_parity_64bit(u64 word)
 541 {
 542         word ^= word >> 1;
 543         word ^= word >> 2;
 544         word ^= word >> 4;
 545         word ^= word >> 8;
 546         word ^= word >> 16;
 547         word ^= word >> 32;
 548         return word & 1;
 549 }
 550
 551 static u64 qat_hal_set_uword_ecc(u64 uword)
 552 {
 553         u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 554                 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 555                 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 556                 bit6_mask = 0xdaf69a46910ULL;
 557
 558         /* clear the ecc bits */
 559         uword &= ~(0x7fULL << 0x2C);
 560         uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 561         uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 562         uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 563         uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 564         uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 565         uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 566         uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 567         return uword;
 568 }
 569
 570 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 571                        unsigned char ae, unsigned int uaddr,
 572                        unsigned int words_num, u64 *uword)
 573 {
 574         unsigned int ustore_addr;
 575         unsigned int i;
 576
 577         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 578         uaddr |= UA_ECS;
 579         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 580         for (i = 0; i < words_num; i++) {
 581                 unsigned int uwrd_lo, uwrd_hi;
 582                 u64 tmp;
 583
 584                 tmp = qat_hal_set_uword_ecc(uword[i]);
 585                 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 586                 uwrd_hi = (unsigned int)(tmp >> 0x20);
 587                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 588                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 589         }
 590         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 591 }
 592
 593 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 594                                unsigned char ae, unsigned int ctx_mask)
 595 {
 596         unsigned int ctx;
 597
 598         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 599         ctx &= IGNORE_W1C_MASK;
 600         ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 601         ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 602         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 603 }
 604
 605 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
 606 {
 607         unsigned long ae_mask = handle->hal_handle->ae_mask;
 608         unsigned char ae;
 609         unsigned short reg;
 610
 611         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 612                 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 613                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 614                                              reg, 0);
 615                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 616                                              reg, 0);
 617                 }
 618         }
 619 }
 620
 621 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 622 {
 623         unsigned long ae_mask = handle->hal_handle->ae_mask;
 624         unsigned char ae;
 625         unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 626         int times = MAX_RETRY_TIMES;
 627         unsigned int csr_val = 0;
 628         unsigned int savctx = 0;
 629         int ret = 0;
 630
 631         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 632                 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 633                 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 634                 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 635                 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 636                 csr_val &= IGNORE_W1C_MASK;
 637                 if (handle->chip_info->nn)
 638                         csr_val |= CE_NN_MODE;
 639
 640                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 641                 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 642                                   (u64 *)inst);
 643                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 644                                     handle->hal_handle->upc_mask &
 645                                     INIT_PC_VALUE);
 646                 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 647                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 648                 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 649                 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 650                                     CTX_SIG_EVENTS_INDIRECT, 0);
 651                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 652                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 653         }
 654         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 655                 /* wait for AE to finish */
 656                 do {
 657                         ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 658                 } while (ret && times--);
 659
 660                 if (times < 0) {
 661                         pr_err("QAT: clear GPR of AE %d failed", ae);
 662                         return -EINVAL;
 663                 }
 664                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 665                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 666                                   savctx & ACS_ACNO);
 667                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 668                                   INIT_CTX_ENABLE_VALUE);
 669                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 670                                     handle->hal_handle->upc_mask &
 671                                     INIT_PC_VALUE);
 672                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 673                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 674                 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 675                                          INIT_WAKEUP_EVENTS_VALUE);
 676                 qat_hal_put_sig_event(handle, ae, ctx_mask,
 677                                       INIT_SIG_EVENTS_VALUE);
 678         }
 679         return 0;
 680 }
 681
 682 static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 683                              struct adf_accel_dev *accel_dev)
 684 {
 685         struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 686         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 687         struct adf_bar *misc_bar =
 688                         &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
 689         unsigned int max_en_ae_id = 0;
 690         struct adf_bar *sram_bar;
 691         unsigned int csr_val = 0;
 692         unsigned long ae_mask;
 693         unsigned char ae = 0;
 694         int ret = 0;
 695
 696         handle->pci_dev = pci_info->pci_dev;
 697         switch (handle->pci_dev->device) {
 698         case ADF_4XXX_PCI_DEVICE_ID:
 699                 handle->chip_info->mmp_sram_size = 0;
 700                 handle->chip_info->nn = false;
 701                 handle->chip_info->lm2lm3 = true;
 702                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
 703                 handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;
 704                 handle->chip_info->icp_rst_mask = 0x100015;
 705                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;
 706                 handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;
 707                 handle->chip_info->wakeup_event_val = 0x80000000;
 708                 handle->chip_info->fw_auth = true;
 709                 handle->chip_info->css_3k = true;
 710                 handle->chip_info->tgroup_share_ustore = true;
 711                 handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
 712                 handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
 713                 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;
 714                 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;
 715                 handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
 716                 handle->chip_info->fcu_loaded_ae_pos = 0;
 717
 718                 handle->hal_cap_g_ctl_csr_addr_v =
 719                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 720                         ICP_QAT_CAP_OFFSET_4XXX);
 721                 handle->hal_cap_ae_xfer_csr_addr_v =
 722                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 723                         ICP_QAT_AE_OFFSET_4XXX);
 724                 handle->hal_ep_csr_addr_v =
 725                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 726                         ICP_QAT_EP_OFFSET_4XXX);
 727                 handle->hal_cap_ae_local_csr_addr_v =
 728                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 729                                                         + LOCAL_TO_XFER_REG_OFFSET);
 730                 break;
 731         case PCI_DEVICE_ID_INTEL_QAT_C62X:
 732         case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
 733                 handle->chip_info->mmp_sram_size = 0;
 734                 handle->chip_info->nn = true;
 735                 handle->chip_info->lm2lm3 = false;
 736                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 737                 handle->chip_info->icp_rst_csr = ICP_RESET;
 738                 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 739                                                   (hw_data->accel_mask << RST_CSR_QAT_LSB);
 740                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 741                 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 742                 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 743                 handle->chip_info->fw_auth = true;
 744                 handle->chip_info->css_3k = false;
 745                 handle->chip_info->tgroup_share_ustore = false;
 746                 handle->chip_info->fcu_ctl_csr = FCU_CONTROL;
 747                 handle->chip_info->fcu_sts_csr = FCU_STATUS;
 748                 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;
 749                 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
 750                 handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
 751                 handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
 752                 handle->hal_cap_g_ctl_csr_addr_v =
 753                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 754                         ICP_QAT_CAP_OFFSET);
 755                 handle->hal_cap_ae_xfer_csr_addr_v =
 756                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 757                         ICP_QAT_AE_OFFSET);
 758                 handle->hal_ep_csr_addr_v =
 759                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 760                         ICP_QAT_EP_OFFSET);
 761                 handle->hal_cap_ae_local_csr_addr_v =
 762                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 763                         + LOCAL_TO_XFER_REG_OFFSET);
 764                 break;
 765         case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
 766                 handle->chip_info->mmp_sram_size = 0x40000;
 767                 handle->chip_info->nn = true;
 768                 handle->chip_info->lm2lm3 = false;
 769                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 770                 handle->chip_info->icp_rst_csr = ICP_RESET;
 771                 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 772                                                   (hw_data->accel_mask << RST_CSR_QAT_LSB);
 773                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 774                 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 775                 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 776                 handle->chip_info->fw_auth = false;
 777                 handle->chip_info->css_3k = false;
 778                 handle->chip_info->tgroup_share_ustore = false;
 779                 handle->chip_info->fcu_ctl_csr = 0;
 780                 handle->chip_info->fcu_sts_csr = 0;
 781                 handle->chip_info->fcu_dram_addr_hi = 0;
 782                 handle->chip_info->fcu_dram_addr_lo = 0;
 783                 handle->chip_info->fcu_loaded_ae_csr = 0;
 784                 handle->chip_info->fcu_loaded_ae_pos = 0;
 785                 handle->hal_cap_g_ctl_csr_addr_v =
 786                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 787                         ICP_QAT_CAP_OFFSET);
 788                 handle->hal_cap_ae_xfer_csr_addr_v =
 789                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 790                         ICP_QAT_AE_OFFSET);
 791                 handle->hal_ep_csr_addr_v =
 792                         (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 793                         ICP_QAT_EP_OFFSET);
 794                 handle->hal_cap_ae_local_csr_addr_v =
 795                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 796                         + LOCAL_TO_XFER_REG_OFFSET);
 797                 break;
 798         default:
 799                 ret = -EINVAL;
 800                 goto out_err;
 801         }
 802
 803         if (handle->chip_info->mmp_sram_size > 0) {
 804                 sram_bar =
 805                         &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
 806                 handle->hal_sram_addr_v = sram_bar->virt_addr;
 807         }
 808         handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 809         handle->hal_handle->ae_mask = hw_data->ae_mask;
 810         handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
 811         handle->hal_handle->slice_mask = hw_data->accel_mask;
 812         handle->cfg_ae_mask = ALL_AE_MASK;
 813         /* create AE objects */
 814         handle->hal_handle->upc_mask = 0x1ffff;
 815         handle->hal_handle->max_ustore = 0x4000;
 816
 817         ae_mask = handle->hal_handle->ae_mask;
 818         for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {
 819                 handle->hal_handle->aes[ae].free_addr = 0;
 820                 handle->hal_handle->aes[ae].free_size =
 821                     handle->hal_handle->max_ustore;
 822                 handle->hal_handle->aes[ae].ustore_size =
 823                     handle->hal_handle->max_ustore;
 824                 handle->hal_handle->aes[ae].live_ctx_mask =
 825                                                 ICP_QAT_UCLO_AE_ALL_CTX;
 826                 max_en_ae_id = ae;
 827         }
 828         handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 829
 830         /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 831         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 832                 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
 833                 csr_val |= 0x1;
 834                 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 835         }
 836 out_err:
 837         return ret;
 838 }
 839
 840 int qat_hal_init(struct adf_accel_dev *accel_dev)
 841 {
 842         struct icp_qat_fw_loader_handle *handle;
 843         int ret = 0;
 844
 845         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 846         if (!handle)
 847                 return -ENOMEM;
 848
 849         handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 850         if (!handle->hal_handle) {
 851                 ret = -ENOMEM;
 852                 goto out_hal_handle;
 853         }
 854
 855         handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);
 856         if (!handle->chip_info) {
 857                 ret = -ENOMEM;
 858                 goto out_chip_info;
 859         }
 860
 861         ret = qat_hal_chip_init(handle, accel_dev);
 862         if (ret) {
 863                 dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");
 864                 goto out_err;
 865         }
 866
 867         /* take all AEs out of reset */
 868         ret = qat_hal_clr_reset(handle);
 869         if (ret) {
 870                 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 871                 goto out_err;
 872         }
 873
 874         qat_hal_clear_xfer(handle);
 875         if (!handle->chip_info->fw_auth) {
 876                 ret = qat_hal_clear_gpr(handle);
 877                 if (ret)
 878                         goto out_err;
 879         }
 880
 881         accel_dev->fw_loader->fw_loader = handle;
 882         return 0;
 883
 884 out_err:
 885         kfree(handle->chip_info);
 886 out_chip_info:
 887         kfree(handle->hal_handle);
 888 out_hal_handle:
 889         kfree(handle);
 890         return ret;
 891 }
 892
 893 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 894 {
 895         if (!handle)
 896                 return;
 897         kfree(handle->chip_info);
 898         kfree(handle->hal_handle);
 899         kfree(handle);
 900 }
 901
 902 int qat_hal_start(struct icp_qat_fw_loader_handle *handle)
 903 {
 904         unsigned long ae_mask = handle->hal_handle->ae_mask;
 905         u32 wakeup_val = handle->chip_info->wakeup_event_val;
 906         u32 fcu_ctl_csr, fcu_sts_csr;
 907         unsigned int fcu_sts;
 908         unsigned char ae;
 909         u32 ae_ctr = 0;
 910         int retry = 0;
 911
 912         if (handle->chip_info->fw_auth) {
 913                 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
 914                 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
 915                 ae_ctr = hweight32(ae_mask);
 916                 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
 917                 do {
 918                         msleep(FW_AUTH_WAIT_PERIOD);
 919                         fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
 920                         if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
 921                                 return ae_ctr;
 922                 } while (retry++ < FW_AUTH_MAX_RETRY);
 923                 pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);
 924                 return 0;
 925         } else {
 926                 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 927                         qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val);
 928                         qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
 929                         ae_ctr++;
 930                 }
 931                 return ae_ctr;
 932         }
 933 }
 934
 935 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 936                   unsigned int ctx_mask)
 937 {
 938         if (!handle->chip_info->fw_auth)
 939                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 940 }
 941
 942 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 943                     unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 944 {
 945         qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 946                             handle->hal_handle->upc_mask & upc);
 947 }
 948
 949 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 950                                unsigned char ae, unsigned int uaddr,
 951                                unsigned int words_num, u64 *uword)
 952 {
 953         unsigned int i, uwrd_lo, uwrd_hi;
 954         unsigned int ustore_addr, misc_control;
 955
 956         misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 957         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 958                           misc_control & 0xfffffffb);
 959         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 960         uaddr |= UA_ECS;
 961         for (i = 0; i < words_num; i++) {
 962                 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 963                 uaddr++;
 964                 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
 965                 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
 966                 uword[i] = uwrd_hi;
 967                 uword[i] = (uword[i] << 0x20) | uwrd_lo;
 968         }
 969         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 970         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 971 }
 972
 973 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 974                      unsigned char ae, unsigned int uaddr,
 975                      unsigned int words_num, unsigned int *data)
 976 {
 977         unsigned int i, ustore_addr;
 978
 979         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 980         uaddr |= UA_ECS;
 981         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 982         for (i = 0; i < words_num; i++) {
 983                 unsigned int uwrd_lo, uwrd_hi, tmp;
 984
 985                 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 986                           ((data[i] & 0xff00) << 2) |
 987                           (0x3 << 8) | (data[i] & 0xff);
 988                 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 989                 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 990                 tmp = ((data[i] >> 0x10) & 0xffff);
 991                 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 992                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 993                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 994         }
 995         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 996 }
 997
 998 #define MAX_EXEC_INST 100
 999 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
1000                                    unsigned char ae, unsigned char ctx,
1001                                    u64 *micro_inst, unsigned int inst_num,
1002                                    int code_off, unsigned int max_cycle,
1003                                    unsigned int *endpc)
1004 {
1005         unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;
1006         unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;
1007         unsigned int ind_t_index = 0, ind_t_index_byte = 0;
1008         unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;
1009         unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;
1010         u64 savuwords[MAX_EXEC_INST];
1011         unsigned int ind_cnt_sig;
1012         unsigned int ind_sig, act_sig;
1013         unsigned int csr_val = 0, newcsr_val;
1014         unsigned int savctx;
1015         unsigned int savcc, wakeup_events, savpc;
1016         unsigned int ctxarb_ctl, ctx_enables;
1017
1018         if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
1019                 pr_err("QAT: invalid instruction num %d\n", inst_num);
1020                 return -EINVAL;
1021         }
1022         /* save current context */
1023         ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
1024         ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
1025         ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
1026                                                 INDIRECT_LM_ADDR_0_BYTE_INDEX);
1027         ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
1028                                                 INDIRECT_LM_ADDR_1_BYTE_INDEX);
1029         if (handle->chip_info->lm2lm3) {
1030                 ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1031                                                    LM_ADDR_2_INDIRECT);
1032                 ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1033                                                    LM_ADDR_3_INDIRECT);
1034                 ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1035                                                         INDIRECT_LM_ADDR_2_BYTE_INDEX);
1036                 ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1037                                                         INDIRECT_LM_ADDR_3_BYTE_INDEX);
1038                 ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,
1039                                                   INDIRECT_T_INDEX);
1040                 ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,
1041                                                        INDIRECT_T_INDEX_BYTE_INDEX);
1042         }
1043         if (inst_num <= MAX_EXEC_INST)
1044                 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
1045         qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
1046         savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
1047         savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1048         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1049         ctx_enables &= IGNORE_W1C_MASK;
1050         savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
1051         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1052         ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1053         ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1054                                           FUTURE_COUNT_SIGNAL_INDIRECT);
1055         ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1056                                       CTX_SIG_EVENTS_INDIRECT);
1057         act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
1058         /* execute micro codes */
1059         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1060         qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
1061         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
1062         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
1063         if (code_off)
1064                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
1065         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
1066         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
1067         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1068         qat_hal_enable_ctx(handle, ae, (1 << ctx));
1069         /* wait for micro codes to finish */
1070         if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
1071                 return -EFAULT;
1072         if (endpc) {
1073                 unsigned int ctx_status;
1074
1075                 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
1076                                                  CTX_STS_INDIRECT);
1077                 *endpc = ctx_status & handle->hal_handle->upc_mask;
1078         }
1079         /* retore to saved context */
1080         qat_hal_disable_ctx(handle, ae, (1 << ctx));
1081         if (inst_num <= MAX_EXEC_INST)
1082                 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
1083         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
1084         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
1085                             handle->hal_handle->upc_mask & savpc);
1086         csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
1087         newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1088         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1089         qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
1090         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
1091         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
1092         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1093                             LM_ADDR_0_INDIRECT, ind_lm_addr0);
1094         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1095                             LM_ADDR_1_INDIRECT, ind_lm_addr1);
1096         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1097                             INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
1098         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1099                             INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
1100         if (handle->chip_info->lm2lm3) {
1101                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT,
1102                                     ind_lm_addr2);
1103                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT,
1104                                     ind_lm_addr3);
1105                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1106                                     INDIRECT_LM_ADDR_2_BYTE_INDEX,
1107                                     ind_lm_addr_byte2);
1108                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1109                                     INDIRECT_LM_ADDR_3_BYTE_INDEX,
1110                                     ind_lm_addr_byte3);
1111                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1112                                     INDIRECT_T_INDEX, ind_t_index);
1113                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1114                                     INDIRECT_T_INDEX_BYTE_INDEX,
1115                                     ind_t_index_byte);
1116         }
1117         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1118                             FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
1119         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1120                             CTX_SIG_EVENTS_INDIRECT, ind_sig);
1121         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
1122         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1123
1124         return 0;
1125 }
1126
1127 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1128                               unsigned char ae, unsigned char ctx,
1129                               enum icp_qat_uof_regtype reg_type,
1130                               unsigned short reg_num, unsigned int *data)
1131 {
1132         unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1133         unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1134         unsigned short reg_addr;
1135         int status = 0;
1136         u64 insts, savuword;
1137
1138         reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1139         if (reg_addr == BAD_REGADDR) {
1140                 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1141                 return -EINVAL;
1142         }
1143         switch (reg_type) {
1144         case ICP_GPA_REL:
1145                 insts = 0xA070000000ull | (reg_addr & 0x3ff);
1146                 break;
1147         default:
1148                 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1149                 break;
1150         }
1151         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1152         ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1153         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1154         ctx_enables &= IGNORE_W1C_MASK;
1155         if (ctx != (savctx & ACS_ACNO))
1156                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1157                                   ctx & ACS_ACNO);
1158         qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
1159         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1160         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
1161         uaddr = UA_ECS;
1162         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1163         insts = qat_hal_set_uword_ecc(insts);
1164         uwrd_lo = (unsigned int)(insts & 0xffffffff);
1165         uwrd_hi = (unsigned int)(insts >> 0x20);
1166         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1167         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1168         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1169         /* delay for at least 8 cycles */
1170         qat_hal_wait_cycles(handle, ae, 0x8, 0);
1171         /*
1172          * read ALU output
1173          * the instruction should have been executed
1174          * prior to clearing the ECS in putUwords
1175          */
1176         *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
1177         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1178         qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1179         if (ctx != (savctx & ACS_ACNO))
1180                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1181                                   savctx & ACS_ACNO);
1182         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1183         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1184
1185         return status;
1186 }
1187
1188 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1189                               unsigned char ae, unsigned char ctx,
1190                               enum icp_qat_uof_regtype reg_type,
1191                               unsigned short reg_num, unsigned int data)
1192 {
1193         unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1194         u64 insts[] = {
1195                 0x0F440000000ull,
1196                 0x0F040000000ull,
1197                 0x0F0000C0300ull,
1198                 0x0E000010000ull
1199         };
1200         const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1201         const int imm_w1 = 0, imm_w0 = 1;
1202
1203         dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1204         if (dest_addr == BAD_REGADDR) {
1205                 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1206                 return -EINVAL;
1207         }
1208
1209         data16lo = 0xffff & data;
1210         data16hi = 0xffff & (data >> 0x10);
1211         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1212                                           (0xff & data16hi));
1213         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1214                                            (0xff & data16lo));
1215         switch (reg_type) {
1216         case ICP_GPA_REL:
1217                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1218                     ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1219                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1220                     ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1221                 break;
1222         default:
1223                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1224                     ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1225
1226                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1227                     ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1228                 break;
1229         }
1230
1231         return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1232                                        code_off, num_inst * 0x5, NULL);
1233 }
1234
1235 int qat_hal_get_ins_num(void)
1236 {
1237         return ARRAY_SIZE(inst_4b);
1238 }
1239
1240 static int qat_hal_concat_micro_code(u64 *micro_inst,
1241                                      unsigned int inst_num, unsigned int size,
1242                                      unsigned int addr, unsigned int *value)
1243 {
1244         int i;
1245         unsigned int cur_value;
1246         const u64 *inst_arr;
1247         int fixup_offset;
1248         int usize = 0;
1249         int orig_num;
1250
1251         orig_num = inst_num;
1252         cur_value = value[0];
1253         inst_arr = inst_4b;
1254         usize = ARRAY_SIZE(inst_4b);
1255         fixup_offset = inst_num;
1256         for (i = 0; i < usize; i++)
1257                 micro_inst[inst_num++] = inst_arr[i];
1258         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1259         fixup_offset++;
1260         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1261         fixup_offset++;
1262         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1263         fixup_offset++;
1264         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1265
1266         return inst_num - orig_num;
1267 }
1268
1269 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1270                                       unsigned char ae, unsigned char ctx,
1271                                       int *pfirst_exec, u64 *micro_inst,
1272                                       unsigned int inst_num)
1273 {
1274         int stat = 0;
1275         unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1276         unsigned int gprb0 = 0, gprb1 = 0;
1277
1278         if (*pfirst_exec) {
1279                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1280                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1281                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1282                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1283                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1284                 *pfirst_exec = 0;
1285         }
1286         stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1287                                        inst_num * 0x5, NULL);
1288         if (stat != 0)
1289                 return -EFAULT;
1290         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1291         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1292         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1293         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1294         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1295
1296         return 0;
1297 }
1298
1299 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1300                         unsigned char ae,
1301                         struct icp_qat_uof_batch_init *lm_init_header)
1302 {
1303         struct icp_qat_uof_batch_init *plm_init;
1304         u64 *micro_inst_arry;
1305         int micro_inst_num;
1306         int alloc_inst_size;
1307         int first_exec = 1;
1308         int stat = 0;
1309
1310         plm_init = lm_init_header->next;
1311         alloc_inst_size = lm_init_header->size;
1312         if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1313                 alloc_inst_size = handle->hal_handle->max_ustore;
1314         micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1315                                         GFP_KERNEL);
1316         if (!micro_inst_arry)
1317                 return -ENOMEM;
1318         micro_inst_num = 0;
1319         while (plm_init) {
1320                 unsigned int addr, *value, size;
1321
1322                 ae = plm_init->ae;
1323                 addr = plm_init->addr;
1324                 value = plm_init->value;
1325                 size = plm_init->size;
1326                 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1327                                                             micro_inst_num,
1328                                                             size, addr, value);
1329                 plm_init = plm_init->next;
1330         }
1331         /* exec micro codes */
1332         if (micro_inst_arry && micro_inst_num > 0) {
1333                 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1334                 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1335                                                   micro_inst_arry,
1336                                                   micro_inst_num);
1337         }
1338         kfree(micro_inst_arry);
1339         return stat;
1340 }
1341
1342 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1343                                    unsigned char ae, unsigned char ctx,
1344                                    enum icp_qat_uof_regtype reg_type,
1345                                    unsigned short reg_num, unsigned int val)
1346 {
1347         int status = 0;
1348         unsigned int reg_addr;
1349         unsigned int ctx_enables;
1350         unsigned short mask;
1351         unsigned short dr_offset = 0x10;
1352
1353         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1354         if (CE_INUSE_CONTEXTS & ctx_enables) {
1355                 if (ctx & 0x1) {
1356                         pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1357                         return -EINVAL;
1358                 }
1359                 mask = 0x1f;
1360                 dr_offset = 0x20;
1361         } else {
1362                 mask = 0x0f;
1363         }
1364         if (reg_num & ~mask)
1365                 return -EINVAL;
1366         reg_addr = reg_num + (ctx << 0x5);
1367         switch (reg_type) {
1368         case ICP_SR_RD_REL:
1369         case ICP_SR_REL:
1370                 SET_AE_XFER(handle, ae, reg_addr, val);
1371                 break;
1372         case ICP_DR_RD_REL:
1373         case ICP_DR_REL:
1374                 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1375                 break;
1376         default:
1377                 status = -EINVAL;
1378                 break;
1379         }
1380         return status;
1381 }
1382
1383 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1384                                    unsigned char ae, unsigned char ctx,
1385                                    enum icp_qat_uof_regtype reg_type,
1386                                    unsigned short reg_num, unsigned int data)
1387 {
1388         unsigned int gprval, ctx_enables;
1389         unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1390             data16low;
1391         unsigned short reg_mask;
1392         int status = 0;
1393         u64 micro_inst[] = {
1394                 0x0F440000000ull,
1395                 0x0F040000000ull,
1396                 0x0A000000000ull,
1397                 0x0F0000C0300ull,
1398                 0x0E000010000ull
1399         };
1400         const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1401         const unsigned short gprnum = 0, dly = num_inst * 0x5;
1402
1403         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1404         if (CE_INUSE_CONTEXTS & ctx_enables) {
1405                 if (ctx & 0x1) {
1406                         pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1407                         return -EINVAL;
1408                 }
1409                 reg_mask = (unsigned short)~0x1f;
1410         } else {
1411                 reg_mask = (unsigned short)~0xf;
1412         }
1413         if (reg_num & reg_mask)
1414                 return -EINVAL;
1415         xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1416         if (xfr_addr == BAD_REGADDR) {
1417                 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1418                 return -EINVAL;
1419         }
1420         status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1421         if (status) {
1422                 pr_err("QAT: failed to read register");
1423                 return status;
1424         }
1425         gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1426         data16low = 0xffff & data;
1427         data16hi = 0xffff & (data >> 0x10);
1428         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1429                                           (unsigned short)(0xff & data16hi));
1430         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1431                                            (unsigned short)(0xff & data16low));
1432         micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1433             ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1434         micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1435             ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1436         micro_inst[0x2] = micro_inst[0x2] |
1437             ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1438         status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1439                                          code_off, dly, NULL);
1440         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1441         return status;
1442 }
1443
1444 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1445                               unsigned char ae, unsigned char ctx,
1446                               unsigned short nn, unsigned int val)
1447 {
1448         unsigned int ctx_enables;
1449         int stat = 0;
1450
1451         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1452         ctx_enables &= IGNORE_W1C_MASK;
1453         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1454
1455         stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1456         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1457         return stat;
1458 }
1459
1460 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1461                                       *handle, unsigned char ae,
1462                                       unsigned short absreg_num,
1463                                       unsigned short *relreg,
1464                                       unsigned char *ctx)
1465 {
1466         unsigned int ctx_enables;
1467
1468         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1469         if (ctx_enables & CE_INUSE_CONTEXTS) {
1470                 /* 4-ctx mode */
1471                 *relreg = absreg_num & 0x1F;
1472                 *ctx = (absreg_num >> 0x4) & 0x6;
1473         } else {
1474                 /* 8-ctx mode */
1475                 *relreg = absreg_num & 0x0F;
1476                 *ctx = (absreg_num >> 0x4) & 0x7;
1477         }
1478         return 0;
1479 }
1480
1481 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1482                      unsigned char ae, unsigned long ctx_mask,
1483                      enum icp_qat_uof_regtype reg_type,
1484                      unsigned short reg_num, unsigned int regdata)
1485 {
1486         int stat = 0;
1487         unsigned short reg;
1488         unsigned char ctx = 0;
1489         enum icp_qat_uof_regtype type;
1490
1491         if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1492                 return -EINVAL;
1493
1494         do {
1495                 if (ctx_mask == 0) {
1496                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1497                                                    &ctx);
1498                         type = reg_type - 1;
1499                 } else {
1500                         reg = reg_num;
1501                         type = reg_type;
1502                         if (!test_bit(ctx, &ctx_mask))
1503                                 continue;
1504                 }
1505                 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1506                 if (stat) {
1507                         pr_err("QAT: write gpr fail\n");
1508                         return -EINVAL;
1509                 }
1510         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1511
1512         return 0;
1513 }
1514
1515 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1516                          unsigned char ae, unsigned long ctx_mask,
1517                          enum icp_qat_uof_regtype reg_type,
1518                          unsigned short reg_num, unsigned int regdata)
1519 {
1520         int stat = 0;
1521         unsigned short reg;
1522         unsigned char ctx = 0;
1523         enum icp_qat_uof_regtype type;
1524
1525         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1526                 return -EINVAL;
1527
1528         do {
1529                 if (ctx_mask == 0) {
1530                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1531                                                    &ctx);
1532                         type = reg_type - 3;
1533                 } else {
1534                         reg = reg_num;
1535                         type = reg_type;
1536                         if (!test_bit(ctx, &ctx_mask))
1537                                 continue;
1538                 }
1539                 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1540                                                regdata);
1541                 if (stat) {
1542                         pr_err("QAT: write wr xfer fail\n");
1543                         return -EINVAL;
1544                 }
1545         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1546
1547         return 0;
1548 }
1549
1550 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1551                          unsigned char ae, unsigned long ctx_mask,
1552                          enum icp_qat_uof_regtype reg_type,
1553                          unsigned short reg_num, unsigned int regdata)
1554 {
1555         int stat = 0;
1556         unsigned short reg;
1557         unsigned char ctx = 0;
1558         enum icp_qat_uof_regtype type;
1559
1560         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1561                 return -EINVAL;
1562
1563         do {
1564                 if (ctx_mask == 0) {
1565                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1566                                                    &ctx);
1567                         type = reg_type - 3;
1568                 } else {
1569                         reg = reg_num;
1570                         type = reg_type;
1571                         if (!test_bit(ctx, &ctx_mask))
1572                                 continue;
1573                 }
1574                 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1575                                                regdata);
1576                 if (stat) {
1577                         pr_err("QAT: write rd xfer fail\n");
1578                         return -EINVAL;
1579                 }
1580         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1581
1582         return 0;
1583 }
1584
1585 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1586                     unsigned char ae, unsigned long ctx_mask,
1587                     unsigned short reg_num, unsigned int regdata)
1588 {
1589         int stat = 0;
1590         unsigned char ctx;
1591         if (!handle->chip_info->nn) {
1592                 dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",
1593                         handle->pci_dev->device);
1594                 return -EINVAL;
1595         }
1596
1597         if (ctx_mask == 0)
1598                 return -EINVAL;
1599
1600         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1601                 if (!test_bit(ctx, &ctx_mask))
1602                         continue;
1603                 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1604                 if (stat) {
1605                         pr_err("QAT: write neigh error\n");
1606                         return -EINVAL;
1607                 }
1608         }
1609
1610         return 0;
1611 }