drivers/crypto/qat/qat_common/qat_hal.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/slab.h>
   4 #include <linux/delay.h>
   5 #include <linux/pci_ids.h>
   6
   7 #include "adf_accel_devices.h"
   8 #include "adf_common_drv.h"
   9 #include "icp_qat_hal.h"
  10 #include "icp_qat_uclo.h"
  11
  12 #define BAD_REGADDR            0xffff
  13 #define MAX_RETRY_TIMES    10000
  14 #define INIT_CTX_ARB_VALUE      0x0
  15 #define INIT_CTX_ENABLE_VALUE     0x0
  16 #define INIT_PC_VALUE        0x0
  17 #define INIT_WAKEUP_EVENTS_VALUE  0x1
  18 #define INIT_SIG_EVENTS_VALUE     0x1
  19 #define INIT_CCENABLE_VALUE       0x2000
  20 #define RST_CSR_QAT_LSB    20
  21 #define RST_CSR_AE_LSB            0
  22 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  23
  24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  25         (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  26         (~(1 << CE_REG_PAR_ERR_BITPOS)))
  27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  28         (inst = ((inst & 0xFFFF00C03FFull) | \
  29                 ((((const_val) << 12) & 0x0FF00000ull) | \
  30                 (((const_val) << 10) & 0x0003FC00ull))))
  31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  32         (inst = ((inst & 0xFFFF00FFF00ull) | \
  33                 ((((const_val) << 12) & 0x0FF00000ull) | \
  34                 (((const_val) <<  0) & 0x000000FFull))))
  35
  36 #define AE(handle, ae) handle->hal_handle->aes[ae]
  37
  38 static const u64 inst_4b[] = {
  39         0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  40         0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  41         0x0A021000000ull
  42 };
  43
  44 static const u64 inst[] = {
  45         0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  46         0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  47         0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  48         0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  49         0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  50         0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  51         0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  52         0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  53         0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  54         0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  55         0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  56         0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  57         0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
  58         0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
  59         0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
  60         0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
  61         0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
  62         0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
  63         0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
  64         0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
  65         0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
  66         0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
  67 };
  68
  69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
  70                           unsigned char ae, unsigned int ctx_mask)
  71 {
  72         AE(handle, ae).live_ctx_mask = ctx_mask;
  73 }
  74
  75 #define CSR_RETRY_TIMES 500
  76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
  77                              unsigned char ae, unsigned int csr)
  78 {
  79         unsigned int iterations = CSR_RETRY_TIMES;
  80         int value;
  81
  82         do {
  83                 value = GET_AE_CSR(handle, ae, csr);
  84                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
  85                         return value;
  86         } while (iterations--);
  87
  88         pr_err("QAT: Read CSR timeout\n");
  89         return 0;
  90 }
  91
  92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
  93                              unsigned char ae, unsigned int csr,
  94                              unsigned int value)
  95 {
  96         unsigned int iterations = CSR_RETRY_TIMES;
  97
  98         do {
  99                 SET_AE_CSR(handle, ae, csr, value);
 100                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 101                         return 0;
 102         } while (iterations--);
 103
 104         pr_err("QAT: Write CSR Timeout\n");
 105         return -EFAULT;
 106 }
 107
 108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 109                                      unsigned char ae, unsigned char ctx,
 110                                      unsigned int *events)
 111 {
 112         unsigned int cur_ctx;
 113
 114         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 115         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 116         *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
 117         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 118 }
 119
 120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 121                                unsigned char ae, unsigned int cycles,
 122                                int chk_inactive)
 123 {
 124         unsigned int base_cnt = 0, cur_cnt = 0;
 125         unsigned int csr = (1 << ACS_ABO_BITPOS);
 126         int times = MAX_RETRY_TIMES;
 127         int elapsed_cycles = 0;
 128
 129         base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 130         base_cnt &= 0xffff;
 131         while ((int)cycles > elapsed_cycles && times--) {
 132                 if (chk_inactive)
 133                         csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 134
 135                 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 136                 cur_cnt &= 0xffff;
 137                 elapsed_cycles = cur_cnt - base_cnt;
 138
 139                 if (elapsed_cycles < 0)
 140                         elapsed_cycles += 0x10000;
 141
 142                 /* ensure at least 8 time cycles elapsed in wait_cycles */
 143                 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 144                         return 0;
 145         }
 146         if (times < 0) {
 147                 pr_err("QAT: wait_num_cycles time out\n");
 148                 return -EFAULT;
 149         }
 150         return 0;
 151 }
 152
 153 #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
 154 #define SET_BIT(wrd, bit) (wrd | 1 << bit)
 155
 156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 157                             unsigned char ae, unsigned char mode)
 158 {
 159         unsigned int csr, new_csr;
 160
 161         if ((mode != 4) && (mode != 8)) {
 162                 pr_err("QAT: bad ctx mode=%d\n", mode);
 163                 return -EINVAL;
 164         }
 165
 166         /* Sets the accelaration engine context mode to either four or eight */
 167         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 168         csr = IGNORE_W1C_MASK & csr;
 169         new_csr = (mode == 4) ?
 170                 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 171                 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 172         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 173         return 0;
 174 }
 175
 176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 177                            unsigned char ae, unsigned char mode)
 178 {
 179         unsigned int csr, new_csr;
 180
 181         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 182         csr &= IGNORE_W1C_MASK;
 183
 184         new_csr = (mode) ?
 185                 SET_BIT(csr, CE_NN_MODE_BITPOS) :
 186                 CLR_BIT(csr, CE_NN_MODE_BITPOS);
 187
 188         if (new_csr != csr)
 189                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 190
 191         return 0;
 192 }
 193
 194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 195                            unsigned char ae, enum icp_qat_uof_regtype lm_type,
 196                            unsigned char mode)
 197 {
 198         unsigned int csr, new_csr;
 199
 200         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 201         csr &= IGNORE_W1C_MASK;
 202         switch (lm_type) {
 203         case ICP_LMEM0:
 204                 new_csr = (mode) ?
 205                         SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 206                         CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 207                 break;
 208         case ICP_LMEM1:
 209                 new_csr = (mode) ?
 210                         SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 211                         CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 212                 break;
 213         default:
 214                 pr_err("QAT: lmType = 0x%x\n", lm_type);
 215                 return -EINVAL;
 216         }
 217
 218         if (new_csr != csr)
 219                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 220         return 0;
 221 }
 222
 223 static unsigned short qat_hal_get_reg_addr(unsigned int type,
 224                                            unsigned short reg_num)
 225 {
 226         unsigned short reg_addr;
 227
 228         switch (type) {
 229         case ICP_GPA_ABS:
 230         case ICP_GPB_ABS:
 231                 reg_addr = 0x80 | (reg_num & 0x7f);
 232                 break;
 233         case ICP_GPA_REL:
 234         case ICP_GPB_REL:
 235                 reg_addr = reg_num & 0x1f;
 236                 break;
 237         case ICP_SR_RD_REL:
 238         case ICP_SR_WR_REL:
 239         case ICP_SR_REL:
 240                 reg_addr = 0x180 | (reg_num & 0x1f);
 241                 break;
 242         case ICP_SR_ABS:
 243                 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 244                 break;
 245         case ICP_DR_RD_REL:
 246         case ICP_DR_WR_REL:
 247         case ICP_DR_REL:
 248                 reg_addr = 0x1c0 | (reg_num & 0x1f);
 249                 break;
 250         case ICP_DR_ABS:
 251                 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 252                 break;
 253         case ICP_NEIGH_REL:
 254                 reg_addr = 0x280 | (reg_num & 0x1f);
 255                 break;
 256         case ICP_LMEM0:
 257                 reg_addr = 0x200;
 258                 break;
 259         case ICP_LMEM1:
 260                 reg_addr = 0x220;
 261                 break;
 262         case ICP_NO_DEST:
 263                 reg_addr = 0x300 | (reg_num & 0xff);
 264                 break;
 265         default:
 266                 reg_addr = BAD_REGADDR;
 267                 break;
 268         }
 269         return reg_addr;
 270 }
 271
 272 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 273 {
 274         unsigned int ae_reset_csr;
 275
 276         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 277         ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
 278         ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
 279         SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 280 }
 281
 282 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 283                                 unsigned char ae, unsigned int ctx_mask,
 284                                 unsigned int ae_csr, unsigned int csr_val)
 285 {
 286         unsigned int ctx, cur_ctx;
 287
 288         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 289
 290         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 291                 if (!(ctx_mask & (1 << ctx)))
 292                         continue;
 293                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 294                 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 295         }
 296
 297         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 298 }
 299
 300 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 301                                 unsigned char ae, unsigned char ctx,
 302                                 unsigned int ae_csr)
 303 {
 304         unsigned int cur_ctx, csr_val;
 305
 306         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 307         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 308         csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
 309         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 310
 311         return csr_val;
 312 }
 313
 314 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 315                                   unsigned char ae, unsigned int ctx_mask,
 316                                   unsigned int events)
 317 {
 318         unsigned int ctx, cur_ctx;
 319
 320         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 321         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 322                 if (!(ctx_mask & (1 << ctx)))
 323                         continue;
 324                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 325                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 326         }
 327         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 328 }
 329
 330 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 331                                      unsigned char ae, unsigned int ctx_mask,
 332                                      unsigned int events)
 333 {
 334         unsigned int ctx, cur_ctx;
 335
 336         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 337         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 338                 if (!(ctx_mask & (1 << ctx)))
 339                         continue;
 340                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 341                 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 342                                   events);
 343         }
 344         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 345 }
 346
 347 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 348 {
 349         unsigned int base_cnt, cur_cnt;
 350         unsigned char ae;
 351         int times = MAX_RETRY_TIMES;
 352
 353         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 354                 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 355                 base_cnt &= 0xffff;
 356
 357                 do {
 358                         cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 359                         cur_cnt &= 0xffff;
 360                 } while (times-- && (cur_cnt == base_cnt));
 361
 362                 if (times < 0) {
 363                         pr_err("QAT: AE%d is inactive!!\n", ae);
 364                         return -EFAULT;
 365                 }
 366         }
 367
 368         return 0;
 369 }
 370
 371 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
 372                             unsigned int ae)
 373 {
 374         unsigned int enable = 0, active = 0;
 375
 376         enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 377         active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 378         if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
 379             (active & (1 << ACS_ABO_BITPOS)))
 380                 return 1;
 381         else
 382                 return 0;
 383 }
 384
 385 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 386 {
 387         unsigned int misc_ctl;
 388         unsigned char ae;
 389
 390         /* stop the timestamp timers */
 391         misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
 392         if (misc_ctl & MC_TIMESTAMP_ENABLE)
 393                 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
 394                             (~MC_TIMESTAMP_ENABLE));
 395
 396         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 397                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 398                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 399         }
 400         /* start timestamp timers */
 401         SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
 402 }
 403
 404 #define ESRAM_AUTO_TINIT        BIT(2)
 405 #define ESRAM_AUTO_TINIT_DONE   BIT(3)
 406 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 407 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 408 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 409 {
 410         void __iomem *csr_addr =
 411                         (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
 412                         ESRAM_AUTO_INIT_CSR_OFFSET);
 413         unsigned int csr_val;
 414         int times = 30;
 415
 416         if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
 417                 return 0;
 418
 419         csr_val = ADF_CSR_RD(csr_addr, 0);
 420         if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 421                 return 0;
 422
 423         csr_val = ADF_CSR_RD(csr_addr, 0);
 424         csr_val |= ESRAM_AUTO_TINIT;
 425         ADF_CSR_WR(csr_addr, 0, csr_val);
 426
 427         do {
 428                 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 429                 csr_val = ADF_CSR_RD(csr_addr, 0);
 430         } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 431         if ((times < 0)) {
 432                 pr_err("QAT: Fail to init eSram!\n");
 433                 return -EFAULT;
 434         }
 435         return 0;
 436 }
 437
 438 #define SHRAM_INIT_CYCLES 2060
 439 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 440 {
 441         unsigned int ae_reset_csr;
 442         unsigned char ae;
 443         unsigned int clk_csr;
 444         unsigned int times = 100;
 445         unsigned int csr;
 446
 447         /* write to the reset csr */
 448         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 449         ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
 450         ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
 451         do {
 452                 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 453                 if (!(times--))
 454                         goto out_err;
 455                 csr = GET_GLB_CSR(handle, ICP_RESET);
 456         } while ((handle->hal_handle->ae_mask |
 457                  (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
 458         /* enable clock */
 459         clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
 460         clk_csr |= handle->hal_handle->ae_mask << 0;
 461         clk_csr |= handle->hal_handle->slice_mask << 20;
 462         SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
 463         if (qat_hal_check_ae_alive(handle))
 464                 goto out_err;
 465
 466         /* Set undefined power-up/reset states to reasonable default values */
 467         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 468                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 469                                   INIT_CTX_ENABLE_VALUE);
 470                 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 471                                     CTX_STS_INDIRECT,
 472                                     handle->hal_handle->upc_mask &
 473                                     INIT_PC_VALUE);
 474                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 475                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 476                 qat_hal_put_wakeup_event(handle, ae,
 477                                          ICP_QAT_UCLO_AE_ALL_CTX,
 478                                          INIT_WAKEUP_EVENTS_VALUE);
 479                 qat_hal_put_sig_event(handle, ae,
 480                                       ICP_QAT_UCLO_AE_ALL_CTX,
 481                                       INIT_SIG_EVENTS_VALUE);
 482         }
 483         if (qat_hal_init_esram(handle))
 484                 goto out_err;
 485         if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 486                 goto out_err;
 487         qat_hal_reset_timestamp(handle);
 488
 489         return 0;
 490 out_err:
 491         pr_err("QAT: failed to get device out of reset\n");
 492         return -EFAULT;
 493 }
 494
 495 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 496                                 unsigned char ae, unsigned int ctx_mask)
 497 {
 498         unsigned int ctx;
 499
 500         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 501         ctx &= IGNORE_W1C_MASK &
 502                 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 503         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 504 }
 505
 506 static u64 qat_hal_parity_64bit(u64 word)
 507 {
 508         word ^= word >> 1;
 509         word ^= word >> 2;
 510         word ^= word >> 4;
 511         word ^= word >> 8;
 512         word ^= word >> 16;
 513         word ^= word >> 32;
 514         return word & 1;
 515 }
 516
 517 static u64 qat_hal_set_uword_ecc(u64 uword)
 518 {
 519         u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 520                 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 521                 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 522                 bit6_mask = 0xdaf69a46910ULL;
 523
 524         /* clear the ecc bits */
 525         uword &= ~(0x7fULL << 0x2C);
 526         uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 527         uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 528         uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 529         uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 530         uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 531         uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 532         uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 533         return uword;
 534 }
 535
 536 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 537                        unsigned char ae, unsigned int uaddr,
 538                        unsigned int words_num, u64 *uword)
 539 {
 540         unsigned int ustore_addr;
 541         unsigned int i;
 542
 543         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 544         uaddr |= UA_ECS;
 545         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 546         for (i = 0; i < words_num; i++) {
 547                 unsigned int uwrd_lo, uwrd_hi;
 548                 u64 tmp;
 549
 550                 tmp = qat_hal_set_uword_ecc(uword[i]);
 551                 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 552                 uwrd_hi = (unsigned int)(tmp >> 0x20);
 553                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 554                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 555         }
 556         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 557 }
 558
 559 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 560                                unsigned char ae, unsigned int ctx_mask)
 561 {
 562         unsigned int ctx;
 563
 564         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 565         ctx &= IGNORE_W1C_MASK;
 566         ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 567         ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 568         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 569 }
 570
 571 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
 572 {
 573         unsigned char ae;
 574         unsigned short reg;
 575
 576         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 577                 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 578                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 579                                              reg, 0);
 580                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 581                                              reg, 0);
 582                 }
 583         }
 584 }
 585
 586 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 587 {
 588         unsigned char ae;
 589         unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 590         int times = MAX_RETRY_TIMES;
 591         unsigned int csr_val = 0;
 592         unsigned int savctx = 0;
 593         int ret = 0;
 594
 595         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 596                 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 597                 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 598                 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 599                 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 600                 csr_val &= IGNORE_W1C_MASK;
 601                 csr_val |= CE_NN_MODE;
 602                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 603                 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 604                                   (u64 *)inst);
 605                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 606                                     handle->hal_handle->upc_mask &
 607                                     INIT_PC_VALUE);
 608                 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 609                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 610                 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 611                 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 612                                     CTX_SIG_EVENTS_INDIRECT, 0);
 613                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 614                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 615         }
 616         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 617                 /* wait for AE to finish */
 618                 do {
 619                         ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 620                 } while (ret && times--);
 621
 622                 if (times < 0) {
 623                         pr_err("QAT: clear GPR of AE %d failed", ae);
 624                         return -EINVAL;
 625                 }
 626                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 627                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 628                                   savctx & ACS_ACNO);
 629                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 630                                   INIT_CTX_ENABLE_VALUE);
 631                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 632                                     handle->hal_handle->upc_mask &
 633                                     INIT_PC_VALUE);
 634                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 635                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 636                 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 637                                          INIT_WAKEUP_EVENTS_VALUE);
 638                 qat_hal_put_sig_event(handle, ae, ctx_mask,
 639                                       INIT_SIG_EVENTS_VALUE);
 640         }
 641         return 0;
 642 }
 643
 644 #define ICP_QAT_AE_OFFSET       0x20000
 645 #define ICP_QAT_CAP_OFFSET       (ICP_QAT_AE_OFFSET + 0x10000)
 646 #define LOCAL_TO_XFER_REG_OFFSET    0x800
 647 #define ICP_QAT_EP_OFFSET       0x3a000
 648 int qat_hal_init(struct adf_accel_dev *accel_dev)
 649 {
 650         unsigned char ae;
 651         unsigned int max_en_ae_id = 0;
 652         struct icp_qat_fw_loader_handle *handle;
 653         struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 654         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 655         struct adf_bar *misc_bar =
 656                         &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
 657         struct adf_bar *sram_bar;
 658
 659         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 660         if (!handle)
 661                 return -ENOMEM;
 662
 663         handle->hal_cap_g_ctl_csr_addr_v =
 664                 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 665                                  ICP_QAT_CAP_OFFSET);
 666         handle->hal_cap_ae_xfer_csr_addr_v =
 667                 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 668                                  ICP_QAT_AE_OFFSET);
 669         handle->hal_ep_csr_addr_v =
 670                 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
 671                                  ICP_QAT_EP_OFFSET);
 672         handle->hal_cap_ae_local_csr_addr_v =
 673                 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
 674                                  LOCAL_TO_XFER_REG_OFFSET);
 675         handle->pci_dev = pci_info->pci_dev;
 676         if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) {
 677                 sram_bar =
 678                         &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
 679                 handle->hal_sram_addr_v = sram_bar->virt_addr;
 680         }
 681         handle->fw_auth = (handle->pci_dev->device ==
 682                            PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true;
 683         handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 684         if (!handle->hal_handle)
 685                 goto out_hal_handle;
 686         handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 687         handle->hal_handle->ae_mask = hw_data->ae_mask;
 688         handle->hal_handle->slice_mask = hw_data->accel_mask;
 689         /* create AE objects */
 690         handle->hal_handle->upc_mask = 0x1ffff;
 691         handle->hal_handle->max_ustore = 0x4000;
 692         for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
 693                 if (!(hw_data->ae_mask & (1 << ae)))
 694                         continue;
 695                 handle->hal_handle->aes[ae].free_addr = 0;
 696                 handle->hal_handle->aes[ae].free_size =
 697                     handle->hal_handle->max_ustore;
 698                 handle->hal_handle->aes[ae].ustore_size =
 699                     handle->hal_handle->max_ustore;
 700                 handle->hal_handle->aes[ae].live_ctx_mask =
 701                                                 ICP_QAT_UCLO_AE_ALL_CTX;
 702                 max_en_ae_id = ae;
 703         }
 704         handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 705         /* take all AEs out of reset */
 706         if (qat_hal_clr_reset(handle)) {
 707                 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 708                 goto out_err;
 709         }
 710         qat_hal_clear_xfer(handle);
 711         if (!handle->fw_auth) {
 712                 if (qat_hal_clear_gpr(handle))
 713                         goto out_err;
 714         }
 715
 716         /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 717         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 718                 unsigned int csr_val = 0;
 719
 720                 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
 721                 csr_val |= 0x1;
 722                 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 723         }
 724         accel_dev->fw_loader->fw_loader = handle;
 725         return 0;
 726
 727 out_err:
 728         kfree(handle->hal_handle);
 729 out_hal_handle:
 730         kfree(handle);
 731         return -EFAULT;
 732 }
 733
 734 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 735 {
 736         if (!handle)
 737                 return;
 738         kfree(handle->hal_handle);
 739         kfree(handle);
 740 }
 741
 742 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 743                    unsigned int ctx_mask)
 744 {
 745         int retry = 0;
 746         unsigned int fcu_sts = 0;
 747
 748         if (handle->fw_auth) {
 749                 SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START);
 750                 do {
 751                         msleep(FW_AUTH_WAIT_PERIOD);
 752                         fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
 753                         if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
 754                                 return;
 755                 } while (retry++ < FW_AUTH_MAX_RETRY);
 756                 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae,
 757                        fcu_sts);
 758         } else {
 759                 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
 760                                  ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
 761                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 762         }
 763 }
 764
 765 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 766                   unsigned int ctx_mask)
 767 {
 768         if (!handle->fw_auth)
 769                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 770 }
 771
 772 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 773                     unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 774 {
 775         qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 776                             handle->hal_handle->upc_mask & upc);
 777 }
 778
 779 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 780                                unsigned char ae, unsigned int uaddr,
 781                                unsigned int words_num, u64 *uword)
 782 {
 783         unsigned int i, uwrd_lo, uwrd_hi;
 784         unsigned int ustore_addr, misc_control;
 785
 786         misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 787         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 788                           misc_control & 0xfffffffb);
 789         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 790         uaddr |= UA_ECS;
 791         for (i = 0; i < words_num; i++) {
 792                 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 793                 uaddr++;
 794                 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
 795                 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
 796                 uword[i] = uwrd_hi;
 797                 uword[i] = (uword[i] << 0x20) | uwrd_lo;
 798         }
 799         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 800         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 801 }
 802
 803 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 804                      unsigned char ae, unsigned int uaddr,
 805                      unsigned int words_num, unsigned int *data)
 806 {
 807         unsigned int i, ustore_addr;
 808
 809         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 810         uaddr |= UA_ECS;
 811         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 812         for (i = 0; i < words_num; i++) {
 813                 unsigned int uwrd_lo, uwrd_hi, tmp;
 814
 815                 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 816                           ((data[i] & 0xff00) << 2) |
 817                           (0x3 << 8) | (data[i] & 0xff);
 818                 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 819                 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 820                 tmp = ((data[i] >> 0x10) & 0xffff);
 821                 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 822                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 823                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 824         }
 825         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 826 }
 827
 828 #define MAX_EXEC_INST 100
 829 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 830                                    unsigned char ae, unsigned char ctx,
 831                                    u64 *micro_inst, unsigned int inst_num,
 832                                    int code_off, unsigned int max_cycle,
 833                                    unsigned int *endpc)
 834 {
 835         u64 savuwords[MAX_EXEC_INST];
 836         unsigned int ind_lm_addr0, ind_lm_addr1;
 837         unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
 838         unsigned int ind_cnt_sig;
 839         unsigned int ind_sig, act_sig;
 840         unsigned int csr_val = 0, newcsr_val;
 841         unsigned int savctx;
 842         unsigned int savcc, wakeup_events, savpc;
 843         unsigned int ctxarb_ctl, ctx_enables;
 844
 845         if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
 846                 pr_err("QAT: invalid instruction num %d\n", inst_num);
 847                 return -EINVAL;
 848         }
 849         /* save current context */
 850         ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
 851         ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
 852         ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
 853                                                 INDIRECT_LM_ADDR_0_BYTE_INDEX);
 854         ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
 855                                                 INDIRECT_LM_ADDR_1_BYTE_INDEX);
 856         if (inst_num <= MAX_EXEC_INST)
 857                 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
 858         qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
 859         savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
 860         savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
 861         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 862         ctx_enables &= IGNORE_W1C_MASK;
 863         savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
 864         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 865         ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
 866         ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
 867                                           FUTURE_COUNT_SIGNAL_INDIRECT);
 868         ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
 869                                       CTX_SIG_EVENTS_INDIRECT);
 870         act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
 871         /* execute micro codes */
 872         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 873         qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
 874         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
 875         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
 876         if (code_off)
 877                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
 878         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
 879         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
 880         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 881         qat_hal_enable_ctx(handle, ae, (1 << ctx));
 882         /* wait for micro codes to finish */
 883         if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
 884                 return -EFAULT;
 885         if (endpc) {
 886                 unsigned int ctx_status;
 887
 888                 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
 889                                                  CTX_STS_INDIRECT);
 890                 *endpc = ctx_status & handle->hal_handle->upc_mask;
 891         }
 892         /* retore to saved context */
 893         qat_hal_disable_ctx(handle, ae, (1 << ctx));
 894         if (inst_num <= MAX_EXEC_INST)
 895                 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
 896         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
 897         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
 898                             handle->hal_handle->upc_mask & savpc);
 899         csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 900         newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
 901         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
 902         qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
 903         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
 904         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
 905         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 906                             LM_ADDR_0_INDIRECT, ind_lm_addr0);
 907         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 908                             LM_ADDR_1_INDIRECT, ind_lm_addr1);
 909         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 910                             INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
 911         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 912                             INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
 913         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 914                             FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
 915         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 916                             CTX_SIG_EVENTS_INDIRECT, ind_sig);
 917         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
 918         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 919
 920         return 0;
 921 }
 922
 923 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
 924                               unsigned char ae, unsigned char ctx,
 925                               enum icp_qat_uof_regtype reg_type,
 926                               unsigned short reg_num, unsigned int *data)
 927 {
 928         unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
 929         unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
 930         unsigned short reg_addr;
 931         int status = 0;
 932         u64 insts, savuword;
 933
 934         reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
 935         if (reg_addr == BAD_REGADDR) {
 936                 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
 937                 return -EINVAL;
 938         }
 939         switch (reg_type) {
 940         case ICP_GPA_REL:
 941                 insts = 0xA070000000ull | (reg_addr & 0x3ff);
 942                 break;
 943         default:
 944                 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
 945                 break;
 946         }
 947         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 948         ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
 949         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 950         ctx_enables &= IGNORE_W1C_MASK;
 951         if (ctx != (savctx & ACS_ACNO))
 952                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 953                                   ctx & ACS_ACNO);
 954         qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
 955         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 956         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 957         uaddr = UA_ECS;
 958         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 959         insts = qat_hal_set_uword_ecc(insts);
 960         uwrd_lo = (unsigned int)(insts & 0xffffffff);
 961         uwrd_hi = (unsigned int)(insts >> 0x20);
 962         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 963         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 964         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 965         /* delay for at least 8 cycles */
 966         qat_hal_wait_cycles(handle, ae, 0x8, 0);
 967         /*
 968          * read ALU output
 969          * the instruction should have been executed
 970          * prior to clearing the ECS in putUwords
 971          */
 972         *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
 973         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 974         qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
 975         if (ctx != (savctx & ACS_ACNO))
 976                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 977                                   savctx & ACS_ACNO);
 978         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
 979         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 980
 981         return status;
 982 }
 983
 984 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
 985                               unsigned char ae, unsigned char ctx,
 986                               enum icp_qat_uof_regtype reg_type,
 987                               unsigned short reg_num, unsigned int data)
 988 {
 989         unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
 990         u64 insts[] = {
 991                 0x0F440000000ull,
 992                 0x0F040000000ull,
 993                 0x0F0000C0300ull,
 994                 0x0E000010000ull
 995         };
 996         const int num_inst = ARRAY_SIZE(insts), code_off = 1;
 997         const int imm_w1 = 0, imm_w0 = 1;
 998
 999         dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1000         if (dest_addr == BAD_REGADDR) {
1001                 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1002                 return -EINVAL;
1003         }
1004
1005         data16lo = 0xffff & data;
1006         data16hi = 0xffff & (data >> 0x10);
1007         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1008                                           (0xff & data16hi));
1009         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1010                                            (0xff & data16lo));
1011         switch (reg_type) {
1012         case ICP_GPA_REL:
1013                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1014                     ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1015                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1016                     ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1017                 break;
1018         default:
1019                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1020                     ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1021
1022                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1023                     ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1024                 break;
1025         }
1026
1027         return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1028                                        code_off, num_inst * 0x5, NULL);
1029 }
1030
1031 int qat_hal_get_ins_num(void)
1032 {
1033         return ARRAY_SIZE(inst_4b);
1034 }
1035
1036 static int qat_hal_concat_micro_code(u64 *micro_inst,
1037                                      unsigned int inst_num, unsigned int size,
1038                                      unsigned int addr, unsigned int *value)
1039 {
1040         int i;
1041         unsigned int cur_value;
1042         const u64 *inst_arr;
1043         int fixup_offset;
1044         int usize = 0;
1045         int orig_num;
1046
1047         orig_num = inst_num;
1048         cur_value = value[0];
1049         inst_arr = inst_4b;
1050         usize = ARRAY_SIZE(inst_4b);
1051         fixup_offset = inst_num;
1052         for (i = 0; i < usize; i++)
1053                 micro_inst[inst_num++] = inst_arr[i];
1054         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1055         fixup_offset++;
1056         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1057         fixup_offset++;
1058         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1059         fixup_offset++;
1060         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1061
1062         return inst_num - orig_num;
1063 }
1064
1065 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1066                                       unsigned char ae, unsigned char ctx,
1067                                       int *pfirst_exec, u64 *micro_inst,
1068                                       unsigned int inst_num)
1069 {
1070         int stat = 0;
1071         unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1072         unsigned int gprb0 = 0, gprb1 = 0;
1073
1074         if (*pfirst_exec) {
1075                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1076                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1077                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1078                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1079                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1080                 *pfirst_exec = 0;
1081         }
1082         stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1083                                        inst_num * 0x5, NULL);
1084         if (stat != 0)
1085                 return -EFAULT;
1086         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1087         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1088         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1089         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1090         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1091
1092         return 0;
1093 }
1094
1095 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1096                         unsigned char ae,
1097                         struct icp_qat_uof_batch_init *lm_init_header)
1098 {
1099         struct icp_qat_uof_batch_init *plm_init;
1100         u64 *micro_inst_arry;
1101         int micro_inst_num;
1102         int alloc_inst_size;
1103         int first_exec = 1;
1104         int stat = 0;
1105
1106         plm_init = lm_init_header->next;
1107         alloc_inst_size = lm_init_header->size;
1108         if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1109                 alloc_inst_size = handle->hal_handle->max_ustore;
1110         micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1111                                         GFP_KERNEL);
1112         if (!micro_inst_arry)
1113                 return -ENOMEM;
1114         micro_inst_num = 0;
1115         while (plm_init) {
1116                 unsigned int addr, *value, size;
1117
1118                 ae = plm_init->ae;
1119                 addr = plm_init->addr;
1120                 value = plm_init->value;
1121                 size = plm_init->size;
1122                 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1123                                                             micro_inst_num,
1124                                                             size, addr, value);
1125                 plm_init = plm_init->next;
1126         }
1127         /* exec micro codes */
1128         if (micro_inst_arry && (micro_inst_num > 0)) {
1129                 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1130                 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1131                                                   micro_inst_arry,
1132                                                   micro_inst_num);
1133         }
1134         kfree(micro_inst_arry);
1135         return stat;
1136 }
1137
1138 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1139                                    unsigned char ae, unsigned char ctx,
1140                                    enum icp_qat_uof_regtype reg_type,
1141                                    unsigned short reg_num, unsigned int val)
1142 {
1143         int status = 0;
1144         unsigned int reg_addr;
1145         unsigned int ctx_enables;
1146         unsigned short mask;
1147         unsigned short dr_offset = 0x10;
1148
1149         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1150         if (CE_INUSE_CONTEXTS & ctx_enables) {
1151                 if (ctx & 0x1) {
1152                         pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1153                         return -EINVAL;
1154                 }
1155                 mask = 0x1f;
1156                 dr_offset = 0x20;
1157         } else {
1158                 mask = 0x0f;
1159         }
1160         if (reg_num & ~mask)
1161                 return -EINVAL;
1162         reg_addr = reg_num + (ctx << 0x5);
1163         switch (reg_type) {
1164         case ICP_SR_RD_REL:
1165         case ICP_SR_REL:
1166                 SET_AE_XFER(handle, ae, reg_addr, val);
1167                 break;
1168         case ICP_DR_RD_REL:
1169         case ICP_DR_REL:
1170                 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1171                 break;
1172         default:
1173                 status = -EINVAL;
1174                 break;
1175         }
1176         return status;
1177 }
1178
1179 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1180                                    unsigned char ae, unsigned char ctx,
1181                                    enum icp_qat_uof_regtype reg_type,
1182                                    unsigned short reg_num, unsigned int data)
1183 {
1184         unsigned int gprval, ctx_enables;
1185         unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1186             data16low;
1187         unsigned short reg_mask;
1188         int status = 0;
1189         u64 micro_inst[] = {
1190                 0x0F440000000ull,
1191                 0x0F040000000ull,
1192                 0x0A000000000ull,
1193                 0x0F0000C0300ull,
1194                 0x0E000010000ull
1195         };
1196         const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1197         const unsigned short gprnum = 0, dly = num_inst * 0x5;
1198
1199         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1200         if (CE_INUSE_CONTEXTS & ctx_enables) {
1201                 if (ctx & 0x1) {
1202                         pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1203                         return -EINVAL;
1204                 }
1205                 reg_mask = (unsigned short)~0x1f;
1206         } else {
1207                 reg_mask = (unsigned short)~0xf;
1208         }
1209         if (reg_num & reg_mask)
1210                 return -EINVAL;
1211         xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1212         if (xfr_addr == BAD_REGADDR) {
1213                 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1214                 return -EINVAL;
1215         }
1216         status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1217         if (status) {
1218                 pr_err("QAT: failed to read register");
1219                 return status;
1220         }
1221         gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1222         data16low = 0xffff & data;
1223         data16hi = 0xffff & (data >> 0x10);
1224         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1225                                           (unsigned short)(0xff & data16hi));
1226         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1227                                            (unsigned short)(0xff & data16low));
1228         micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1229             ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1230         micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1231             ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1232         micro_inst[0x2] = micro_inst[0x2] |
1233             ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1234         status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1235                                          code_off, dly, NULL);
1236         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1237         return status;
1238 }
1239
1240 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1241                               unsigned char ae, unsigned char ctx,
1242                               unsigned short nn, unsigned int val)
1243 {
1244         unsigned int ctx_enables;
1245         int stat = 0;
1246
1247         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1248         ctx_enables &= IGNORE_W1C_MASK;
1249         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1250
1251         stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1252         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1253         return stat;
1254 }
1255
1256 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1257                                       *handle, unsigned char ae,
1258                                       unsigned short absreg_num,
1259                                       unsigned short *relreg,
1260                                       unsigned char *ctx)
1261 {
1262         unsigned int ctx_enables;
1263
1264         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1265         if (ctx_enables & CE_INUSE_CONTEXTS) {
1266                 /* 4-ctx mode */
1267                 *relreg = absreg_num & 0x1F;
1268                 *ctx = (absreg_num >> 0x4) & 0x6;
1269         } else {
1270                 /* 8-ctx mode */
1271                 *relreg = absreg_num & 0x0F;
1272                 *ctx = (absreg_num >> 0x4) & 0x7;
1273         }
1274         return 0;
1275 }
1276
1277 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1278                      unsigned char ae, unsigned char ctx_mask,
1279                      enum icp_qat_uof_regtype reg_type,
1280                      unsigned short reg_num, unsigned int regdata)
1281 {
1282         int stat = 0;
1283         unsigned short reg;
1284         unsigned char ctx = 0;
1285         enum icp_qat_uof_regtype type;
1286
1287         if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1288                 return -EINVAL;
1289
1290         do {
1291                 if (ctx_mask == 0) {
1292                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1293                                                    &ctx);
1294                         type = reg_type - 1;
1295                 } else {
1296                         reg = reg_num;
1297                         type = reg_type;
1298                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1299                                 continue;
1300                 }
1301                 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1302                 if (stat) {
1303                         pr_err("QAT: write gpr fail\n");
1304                         return -EINVAL;
1305                 }
1306         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1307
1308         return 0;
1309 }
1310
1311 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1312                          unsigned char ae, unsigned char ctx_mask,
1313                          enum icp_qat_uof_regtype reg_type,
1314                          unsigned short reg_num, unsigned int regdata)
1315 {
1316         int stat = 0;
1317         unsigned short reg;
1318         unsigned char ctx = 0;
1319         enum icp_qat_uof_regtype type;
1320
1321         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1322                 return -EINVAL;
1323
1324         do {
1325                 if (ctx_mask == 0) {
1326                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1327                                                    &ctx);
1328                         type = reg_type - 3;
1329                 } else {
1330                         reg = reg_num;
1331                         type = reg_type;
1332                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1333                                 continue;
1334                 }
1335                 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1336                                                regdata);
1337                 if (stat) {
1338                         pr_err("QAT: write wr xfer fail\n");
1339                         return -EINVAL;
1340                 }
1341         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1342
1343         return 0;
1344 }
1345
1346 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1347                          unsigned char ae, unsigned char ctx_mask,
1348                          enum icp_qat_uof_regtype reg_type,
1349                          unsigned short reg_num, unsigned int regdata)
1350 {
1351         int stat = 0;
1352         unsigned short reg;
1353         unsigned char ctx = 0;
1354         enum icp_qat_uof_regtype type;
1355
1356         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1357                 return -EINVAL;
1358
1359         do {
1360                 if (ctx_mask == 0) {
1361                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1362                                                    &ctx);
1363                         type = reg_type - 3;
1364                 } else {
1365                         reg = reg_num;
1366                         type = reg_type;
1367                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1368                                 continue;
1369                 }
1370                 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1371                                                regdata);
1372                 if (stat) {
1373                         pr_err("QAT: write rd xfer fail\n");
1374                         return -EINVAL;
1375                 }
1376         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1377
1378         return 0;
1379 }
1380
1381 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1382                     unsigned char ae, unsigned char ctx_mask,
1383                     unsigned short reg_num, unsigned int regdata)
1384 {
1385         int stat = 0;
1386         unsigned char ctx;
1387
1388         if (ctx_mask == 0)
1389                 return -EINVAL;
1390
1391         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1392                 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1393                         continue;
1394                 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1395                 if (stat) {
1396                         pr_err("QAT: write neigh error\n");
1397                         return -EINVAL;
1398                 }
1399         }
1400
1401         return 0;
1402 }