drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2021 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 #if defined(writeq) && defined(CONFIG_64BIT)
  14 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  15 {
  16         writeq(b, addr);
  17 }
  18 #else
  19 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  20 {
  21         __u64 data_out = b;
  22
  23         writel((u32)(data_out), addr);
  24         writel((u32)(data_out >> 32), (addr + 4));
  25 }
  26 #endif
  27
  28 static inline bool
  29 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  30 {
  31         u16 pi, ci, max_entries;
  32         bool is_qfull = false;
  33
  34         pi = op_req_q->pi;
  35         ci = READ_ONCE(op_req_q->ci);
  36         max_entries = op_req_q->num_requests;
  37
  38         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  39                 is_qfull = true;
  40
  41         return is_qfull;
  42 }
  43
  44 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  45 {
  46         u16 i, max_vectors;
  47
  48         max_vectors = mrioc->intr_info_count;
  49
  50         for (i = 0; i < max_vectors; i++)
  51                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  52 }
  53
  54 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  55 {
  56         mrioc->intr_enabled = 0;
  57         mpi3mr_sync_irqs(mrioc);
  58 }
  59
  60 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  61 {
  62         mrioc->intr_enabled = 1;
  63 }
  64
  65 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  66 {
  67         u16 i;
  68
  69         mpi3mr_ioc_disable_intr(mrioc);
  70
  71         if (!mrioc->intr_info)
  72                 return;
  73
  74         for (i = 0; i < mrioc->intr_info_count; i++)
  75                 free_irq(pci_irq_vector(mrioc->pdev, i),
  76                     (mrioc->intr_info + i));
  77
  78         kfree(mrioc->intr_info);
  79         mrioc->intr_info = NULL;
  80         mrioc->intr_info_count = 0;
  81         pci_free_irq_vectors(mrioc->pdev);
  82 }
  83
  84 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  85         dma_addr_t dma_addr)
  86 {
  87         struct mpi3_sge_common *sgel = paddr;
  88
  89         sgel->flags = flags;
  90         sgel->length = cpu_to_le32(length);
  91         sgel->address = cpu_to_le64(dma_addr);
  92 }
  93
  94 void mpi3mr_build_zero_len_sge(void *paddr)
  95 {
  96         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
  97
  98         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
  99 }
 100
 101 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 102         dma_addr_t phys_addr)
 103 {
 104         if (!phys_addr)
 105                 return NULL;
 106
 107         if ((phys_addr < mrioc->reply_buf_dma) ||
 108             (phys_addr > mrioc->reply_buf_dma_max_address))
 109                 return NULL;
 110
 111         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 112 }
 113
 114 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 121 }
 122
 123 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 124         u64 reply_dma)
 125 {
 126         u32 old_idx = 0;
 127
 128         spin_lock(&mrioc->reply_free_queue_lock);
 129         old_idx  =  mrioc->reply_free_queue_host_index;
 130         mrioc->reply_free_queue_host_index = (
 131             (mrioc->reply_free_queue_host_index ==
 132             (mrioc->reply_free_qsz - 1)) ? 0 :
 133             (mrioc->reply_free_queue_host_index + 1));
 134         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 135         writel(mrioc->reply_free_queue_host_index,
 136             &mrioc->sysif_regs->reply_free_host_index);
 137         spin_unlock(&mrioc->reply_free_queue_lock);
 138 }
 139
 140 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 141         u64 sense_buf_dma)
 142 {
 143         u32 old_idx = 0;
 144
 145         spin_lock(&mrioc->sbq_lock);
 146         old_idx  =  mrioc->sbq_host_index;
 147         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 148             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 149             (mrioc->sbq_host_index + 1));
 150         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 151         writel(mrioc->sbq_host_index,
 152             &mrioc->sysif_regs->sense_buffer_free_host_index);
 153         spin_unlock(&mrioc->sbq_lock);
 154 }
 155
 156 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 157         struct mpi3_event_notification_reply *event_reply)
 158 {
 159         char *desc = NULL;
 160         u16 event;
 161
 162         event = event_reply->event;
 163
 164         switch (event) {
 165         case MPI3_EVENT_LOG_DATA:
 166                 desc = "Log Data";
 167                 break;
 168         case MPI3_EVENT_CHANGE:
 169                 desc = "Event Change";
 170                 break;
 171         case MPI3_EVENT_GPIO_INTERRUPT:
 172                 desc = "GPIO Interrupt";
 173                 break;
 174         case MPI3_EVENT_TEMP_THRESHOLD:
 175                 desc = "Temperature Threshold";
 176                 break;
 177         case MPI3_EVENT_CABLE_MGMT:
 178                 desc = "Cable Management";
 179                 break;
 180         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 181                 desc = "Energy Pack Change";
 182                 break;
 183         case MPI3_EVENT_DEVICE_ADDED:
 184         {
 185                 struct mpi3_device_page0 *event_data =
 186                     (struct mpi3_device_page0 *)event_reply->event_data;
 187                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 188                     event_data->dev_handle, event_data->device_form);
 189                 return;
 190         }
 191         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 192         {
 193                 struct mpi3_device_page0 *event_data =
 194                     (struct mpi3_device_page0 *)event_reply->event_data;
 195                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 196                     event_data->dev_handle, event_data->device_form);
 197                 return;
 198         }
 199         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 200         {
 201                 struct mpi3_event_data_device_status_change *event_data =
 202                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 203                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 204                     event_data->dev_handle, event_data->reason_code);
 205                 return;
 206         }
 207         case MPI3_EVENT_SAS_DISCOVERY:
 208         {
 209                 struct mpi3_event_data_sas_discovery *event_data =
 210                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 211                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 212                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 213                     "start" : "stop",
 214                     le32_to_cpu(event_data->discovery_status));
 215                 return;
 216         }
 217         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 218                 desc = "SAS Broadcast Primitive";
 219                 break;
 220         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 221                 desc = "SAS Notify Primitive";
 222                 break;
 223         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 224                 desc = "SAS Init Device Status Change";
 225                 break;
 226         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 227                 desc = "SAS Init Table Overflow";
 228                 break;
 229         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 230                 desc = "SAS Topology Change List";
 231                 break;
 232         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 233                 desc = "Enclosure Device Status Change";
 234                 break;
 235         case MPI3_EVENT_HARD_RESET_RECEIVED:
 236                 desc = "Hard Reset Received";
 237                 break;
 238         case MPI3_EVENT_SAS_PHY_COUNTER:
 239                 desc = "SAS PHY Counter";
 240                 break;
 241         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 242                 desc = "SAS Device Discovery Error";
 243                 break;
 244         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 245                 desc = "PCIE Topology Change List";
 246                 break;
 247         case MPI3_EVENT_PCIE_ENUMERATION:
 248         {
 249                 struct mpi3_event_data_pcie_enumeration *event_data =
 250                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 251                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 252                     (event_data->reason_code ==
 253                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 254                 if (event_data->enumeration_status)
 255                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 256                             le32_to_cpu(event_data->enumeration_status));
 257                 return;
 258         }
 259         case MPI3_EVENT_PREPARE_FOR_RESET:
 260                 desc = "Prepare For Reset";
 261                 break;
 262         }
 263
 264         if (!desc)
 265                 return;
 266
 267         ioc_info(mrioc, "%s\n", desc);
 268 }
 269
 270 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 271         struct mpi3_default_reply *def_reply)
 272 {
 273         struct mpi3_event_notification_reply *event_reply =
 274             (struct mpi3_event_notification_reply *)def_reply;
 275
 276         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 277         mpi3mr_print_event_data(mrioc, event_reply);
 278         mpi3mr_os_handle_events(mrioc, event_reply);
 279 }
 280
 281 static struct mpi3mr_drv_cmd *
 282 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 283         struct mpi3_default_reply *def_reply)
 284 {
 285         u16 idx;
 286
 287         switch (host_tag) {
 288         case MPI3MR_HOSTTAG_INITCMDS:
 289                 return &mrioc->init_cmds;
 290         case MPI3MR_HOSTTAG_BLK_TMS:
 291                 return &mrioc->host_tm_cmds;
 292         case MPI3MR_HOSTTAG_INVALID:
 293                 if (def_reply && def_reply->function ==
 294                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 295                         mpi3mr_handle_events(mrioc, def_reply);
 296                 return NULL;
 297         default:
 298                 break;
 299         }
 300         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 301             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 302                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 303                 return &mrioc->dev_rmhs_cmds[idx];
 304         }
 305
 306         return NULL;
 307 }
 308
 309 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 310         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 311 {
 312         u16 reply_desc_type, host_tag = 0;
 313         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 314         u32 ioc_loginfo = 0;
 315         struct mpi3_status_reply_descriptor *status_desc;
 316         struct mpi3_address_reply_descriptor *addr_desc;
 317         struct mpi3_success_reply_descriptor *success_desc;
 318         struct mpi3_default_reply *def_reply = NULL;
 319         struct mpi3mr_drv_cmd *cmdptr = NULL;
 320         struct mpi3_scsi_io_reply *scsi_reply;
 321         u8 *sense_buf = NULL;
 322
 323         *reply_dma = 0;
 324         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 325             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 326         switch (reply_desc_type) {
 327         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 328                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 329                 host_tag = le16_to_cpu(status_desc->host_tag);
 330                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 331                 if (ioc_status &
 332                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 333                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 334                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 335                 break;
 336         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 337                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 338                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 339                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 340                 if (!def_reply)
 341                         goto out;
 342                 host_tag = le16_to_cpu(def_reply->host_tag);
 343                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 344                 if (ioc_status &
 345                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 346                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 347                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 348                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 349                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 350                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 351                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 352                 }
 353                 break;
 354         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 355                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 356                 host_tag = le16_to_cpu(success_desc->host_tag);
 357                 break;
 358         default:
 359                 break;
 360         }
 361
 362         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 363         if (cmdptr) {
 364                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 365                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 366                         cmdptr->ioc_loginfo = ioc_loginfo;
 367                         cmdptr->ioc_status = ioc_status;
 368                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 369                         if (def_reply) {
 370                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 371                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 372                                     mrioc->facts.reply_sz);
 373                         }
 374                         if (cmdptr->is_waiting) {
 375                                 complete(&cmdptr->done);
 376                                 cmdptr->is_waiting = 0;
 377                         } else if (cmdptr->callback)
 378                                 cmdptr->callback(mrioc, cmdptr);
 379                 }
 380         }
 381 out:
 382         if (sense_buf)
 383                 mpi3mr_repost_sense_buf(mrioc,
 384                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 385 }
 386
 387 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 388 {
 389         u32 exp_phase = mrioc->admin_reply_ephase;
 390         u32 admin_reply_ci = mrioc->admin_reply_ci;
 391         u32 num_admin_replies = 0;
 392         u64 reply_dma = 0;
 393         struct mpi3_default_reply_descriptor *reply_desc;
 394
 395         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 396             admin_reply_ci;
 397
 398         if ((le16_to_cpu(reply_desc->reply_flags) &
 399             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 400                 return 0;
 401
 402         do {
 403                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 404                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 405                 if (reply_dma)
 406                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 407                 num_admin_replies++;
 408                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 409                         admin_reply_ci = 0;
 410                         exp_phase ^= 1;
 411                 }
 412                 reply_desc =
 413                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 414                     admin_reply_ci;
 415                 if ((le16_to_cpu(reply_desc->reply_flags) &
 416                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 417                         break;
 418         } while (1);
 419
 420         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 421         mrioc->admin_reply_ci = admin_reply_ci;
 422         mrioc->admin_reply_ephase = exp_phase;
 423
 424         return num_admin_replies;
 425 }
 426
 427 /**
 428  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 429  *      queue's consumer index from operational reply descriptor queue.
 430  * @op_reply_q: op_reply_qinfo object
 431  * @reply_ci: operational reply descriptor's queue consumer index
 432  *
 433  * Returns reply descriptor frame address
 434  */
 435 static inline struct mpi3_default_reply_descriptor *
 436 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 437 {
 438         void *segment_base_addr;
 439         struct segments *segments = op_reply_q->q_segments;
 440         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 441
 442         segment_base_addr =
 443             segments[reply_ci / op_reply_q->segment_qd].segment;
 444         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 445             (reply_ci % op_reply_q->segment_qd);
 446         return reply_desc;
 447 }
 448
 449 static int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 450         struct mpi3mr_intr_info *intr_info)
 451 {
 452         struct op_reply_qinfo *op_reply_q = intr_info->op_reply_q;
 453         struct op_req_qinfo *op_req_q;
 454         u32 exp_phase;
 455         u32 reply_ci;
 456         u32 num_op_reply = 0;
 457         u64 reply_dma = 0;
 458         struct mpi3_default_reply_descriptor *reply_desc;
 459         u16 req_q_idx = 0, reply_qidx;
 460
 461         reply_qidx = op_reply_q->qid - 1;
 462
 463         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 464                 return 0;
 465
 466         exp_phase = op_reply_q->ephase;
 467         reply_ci = op_reply_q->ci;
 468
 469         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 470         if ((le16_to_cpu(reply_desc->reply_flags) &
 471             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 472                 atomic_dec(&op_reply_q->in_use);
 473                 return 0;
 474         }
 475
 476         do {
 477                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 478                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 479
 480                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 481                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 482                     reply_qidx);
 483                 atomic_dec(&op_reply_q->pend_ios);
 484                 if (reply_dma)
 485                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 486                 num_op_reply++;
 487
 488                 if (++reply_ci == op_reply_q->num_replies) {
 489                         reply_ci = 0;
 490                         exp_phase ^= 1;
 491                 }
 492
 493                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 494
 495                 if ((le16_to_cpu(reply_desc->reply_flags) &
 496                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 497                         break;
 498                 /*
 499                  * Exit completion loop to avoid CPU lockup
 500                  * Ensure remaining completion happens from threaded ISR.
 501                  */
 502                 if (num_op_reply > mrioc->max_host_ios) {
 503                         intr_info->op_reply_q->enable_irq_poll = true;
 504                         break;
 505                 }
 506
 507         } while (1);
 508
 509         writel(reply_ci,
 510             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 511         op_reply_q->ci = reply_ci;
 512         op_reply_q->ephase = exp_phase;
 513
 514         atomic_dec(&op_reply_q->in_use);
 515         return num_op_reply;
 516 }
 517
 518 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 519 {
 520         struct mpi3mr_intr_info *intr_info = privdata;
 521         struct mpi3mr_ioc *mrioc;
 522         u16 midx;
 523         u32 num_admin_replies = 0, num_op_reply = 0;
 524
 525         if (!intr_info)
 526                 return IRQ_NONE;
 527
 528         mrioc = intr_info->mrioc;
 529
 530         if (!mrioc->intr_enabled)
 531                 return IRQ_NONE;
 532
 533         midx = intr_info->msix_index;
 534
 535         if (!midx)
 536                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 537         if (intr_info->op_reply_q)
 538                 num_op_reply = mpi3mr_process_op_reply_q(mrioc, intr_info);
 539
 540         if (num_admin_replies || num_op_reply)
 541                 return IRQ_HANDLED;
 542         else
 543                 return IRQ_NONE;
 544 }
 545
 546 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 547 {
 548         struct mpi3mr_intr_info *intr_info = privdata;
 549         struct mpi3mr_ioc *mrioc;
 550         u16 midx;
 551         int ret;
 552
 553         if (!intr_info)
 554                 return IRQ_NONE;
 555
 556         mrioc = intr_info->mrioc;
 557         midx = intr_info->msix_index;
 558         /* Call primary ISR routine */
 559         ret = mpi3mr_isr_primary(irq, privdata);
 560
 561         /*
 562          * If more IOs are expected, schedule IRQ polling thread.
 563          * Otherwise exit from ISR.
 564          */
 565         if (!intr_info->op_reply_q)
 566                 return ret;
 567
 568         if (!intr_info->op_reply_q->enable_irq_poll ||
 569             !atomic_read(&intr_info->op_reply_q->pend_ios))
 570                 return ret;
 571
 572         disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx));
 573
 574         return IRQ_WAKE_THREAD;
 575 }
 576
 577 /**
 578  * mpi3mr_isr_poll - Reply queue polling routine
 579  * @irq: IRQ
 580  * @privdata: Interrupt info
 581  *
 582  * poll for pending I/O completions in a loop until pending I/Os
 583  * present or controller queue depth I/Os are processed.
 584  *
 585  * Return: IRQ_NONE or IRQ_HANDLED
 586  */
 587 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 588 {
 589         struct mpi3mr_intr_info *intr_info = privdata;
 590         struct mpi3mr_ioc *mrioc;
 591         u16 midx;
 592         u32 num_op_reply = 0;
 593
 594         if (!intr_info || !intr_info->op_reply_q)
 595                 return IRQ_NONE;
 596
 597         mrioc = intr_info->mrioc;
 598         midx = intr_info->msix_index;
 599
 600         /* Poll for pending IOs completions */
 601         do {
 602                 if (!mrioc->intr_enabled)
 603                         break;
 604
 605                 if (!midx)
 606                         mpi3mr_process_admin_reply_q(mrioc);
 607                 if (intr_info->op_reply_q)
 608                         num_op_reply +=
 609                             mpi3mr_process_op_reply_q(mrioc, intr_info);
 610
 611                 usleep_range(mrioc->irqpoll_sleep, 10 * mrioc->irqpoll_sleep);
 612
 613         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 614             (num_op_reply < mrioc->max_host_ios));
 615
 616         intr_info->op_reply_q->enable_irq_poll = false;
 617         enable_irq(pci_irq_vector(mrioc->pdev, midx));
 618
 619         return IRQ_HANDLED;
 620 }
 621
 622 /**
 623  * mpi3mr_request_irq - Request IRQ and register ISR
 624  * @mrioc: Adapter instance reference
 625  * @index: IRQ vector index
 626  *
 627  * Request threaded ISR with primary ISR and secondary
 628  *
 629  * Return: 0 on success and non zero on failures.
 630  */
 631 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 632 {
 633         struct pci_dev *pdev = mrioc->pdev;
 634         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 635         int retval = 0;
 636
 637         intr_info->mrioc = mrioc;
 638         intr_info->msix_index = index;
 639         intr_info->op_reply_q = NULL;
 640
 641         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 642             mrioc->driver_name, mrioc->id, index);
 643
 644         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 645             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 646         if (retval) {
 647                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 648                     intr_info->name, pci_irq_vector(pdev, index));
 649                 return retval;
 650         }
 651
 652         return retval;
 653 }
 654
 655 /**
 656  * mpi3mr_setup_isr - Setup ISR for the controller
 657  * @mrioc: Adapter instance reference
 658  * @setup_one: Request one IRQ or more
 659  *
 660  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 661  *
 662  * Return: 0 on success and non zero on failures.
 663  */
 664 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 665 {
 666         unsigned int irq_flags = PCI_IRQ_MSIX;
 667         int max_vectors;
 668         int retval;
 669         int i;
 670         struct irq_affinity desc = { .pre_vectors =  1};
 671
 672         mpi3mr_cleanup_isr(mrioc);
 673
 674         if (setup_one || reset_devices)
 675                 max_vectors = 1;
 676         else {
 677                 max_vectors =
 678                     min_t(int, mrioc->cpu_count + 1, mrioc->msix_count);
 679
 680                 ioc_info(mrioc,
 681                     "MSI-X vectors supported: %d, no of cores: %d,",
 682                     mrioc->msix_count, mrioc->cpu_count);
 683                 ioc_info(mrioc,
 684                     "MSI-x vectors requested: %d\n", max_vectors);
 685         }
 686
 687         irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 688
 689         mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 690         retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 691                                 1, max_vectors, irq_flags, &desc);
 692         if (retval < 0) {
 693                 ioc_err(mrioc, "Cannot alloc irq vectors\n");
 694                 goto out_failed;
 695         }
 696         if (retval != max_vectors) {
 697                 ioc_info(mrioc,
 698                     "allocated vectors (%d) are less than configured (%d)\n",
 699                     retval, max_vectors);
 700                 /*
 701                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 702                  * between Admin queue and operational queue
 703                  */
 704                 if (retval == 1)
 705                         mrioc->op_reply_q_offset = 0;
 706
 707                 max_vectors = retval;
 708         }
 709         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 710             GFP_KERNEL);
 711         if (!mrioc->intr_info) {
 712                 retval = -ENOMEM;
 713                 pci_free_irq_vectors(mrioc->pdev);
 714                 goto out_failed;
 715         }
 716         for (i = 0; i < max_vectors; i++) {
 717                 retval = mpi3mr_request_irq(mrioc, i);
 718                 if (retval) {
 719                         mrioc->intr_info_count = i;
 720                         goto out_failed;
 721                 }
 722         }
 723         mrioc->intr_info_count = max_vectors;
 724         mpi3mr_ioc_enable_intr(mrioc);
 725         return 0;
 726
 727 out_failed:
 728         mpi3mr_cleanup_isr(mrioc);
 729
 730         return retval;
 731 }
 732
 733 static const struct {
 734         enum mpi3mr_iocstate value;
 735         char *name;
 736 } mrioc_states[] = {
 737         { MRIOC_STATE_READY, "ready" },
 738         { MRIOC_STATE_FAULT, "fault" },
 739         { MRIOC_STATE_RESET, "reset" },
 740         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 741         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 742         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 743 };
 744
 745 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 746 {
 747         int i;
 748         char *name = NULL;
 749
 750         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 751                 if (mrioc_states[i].value == mrioc_state) {
 752                         name = mrioc_states[i].name;
 753                         break;
 754                 }
 755         }
 756         return name;
 757 }
 758
 759 /* Reset reason to name mapper structure*/
 760 static const struct {
 761         enum mpi3mr_reset_reason value;
 762         char *name;
 763 } mpi3mr_reset_reason_codes[] = {
 764         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 765         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 766         { MPI3MR_RESET_FROM_IOCTL, "application invocation" },
 767         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 768         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 769         { MPI3MR_RESET_FROM_IOCTL_TIMEOUT, "IOCTL timeout" },
 770         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 771         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 772         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 773         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 774         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 775         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 776         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 777         {
 778                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 779                 "create request queue timeout"
 780         },
 781         {
 782                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 783                 "create reply queue timeout"
 784         },
 785         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 786         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 787         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 788         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 789         {
 790                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 791                 "component image activation timeout"
 792         },
 793         {
 794                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 795                 "get package version timeout"
 796         },
 797         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 798         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 799 };
 800
 801 /**
 802  * mpi3mr_reset_rc_name - get reset reason code name
 803  * @reason_code: reset reason code value
 804  *
 805  * Map reset reason to an NULL terminated ASCII string
 806  *
 807  * Return: name corresponding to reset reason value or NULL.
 808  */
 809 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 810 {
 811         int i;
 812         char *name = NULL;
 813
 814         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 815                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 816                         name = mpi3mr_reset_reason_codes[i].name;
 817                         break;
 818                 }
 819         }
 820         return name;
 821 }
 822
 823 /* Reset type to name mapper structure*/
 824 static const struct {
 825         u16 reset_type;
 826         char *name;
 827 } mpi3mr_reset_types[] = {
 828         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 829         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 830 };
 831
 832 /**
 833  * mpi3mr_reset_type_name - get reset type name
 834  * @reset_type: reset type value
 835  *
 836  * Map reset type to an NULL terminated ASCII string
 837  *
 838  * Return: name corresponding to reset type value or NULL.
 839  */
 840 static const char *mpi3mr_reset_type_name(u16 reset_type)
 841 {
 842         int i;
 843         char *name = NULL;
 844
 845         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 846                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 847                         name = mpi3mr_reset_types[i].name;
 848                         break;
 849                 }
 850         }
 851         return name;
 852 }
 853
 854 /**
 855  * mpi3mr_print_fault_info - Display fault information
 856  * @mrioc: Adapter instance reference
 857  *
 858  * Display the controller fault information if there is a
 859  * controller fault.
 860  *
 861  * Return: Nothing.
 862  */
 863 static void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 864 {
 865         u32 ioc_status, code, code1, code2, code3;
 866
 867         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 868
 869         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
 870                 code = readl(&mrioc->sysif_regs->fault);
 871                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
 872                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
 873                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
 874
 875                 ioc_info(mrioc,
 876                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
 877                     code, code1, code2, code3);
 878         }
 879 }
 880
 881 /**
 882  * mpi3mr_get_iocstate - Get IOC State
 883  * @mrioc: Adapter instance reference
 884  *
 885  * Return a proper IOC state enum based on the IOC status and
 886  * IOC configuration and unrcoverable state of the controller.
 887  *
 888  * Return: Current IOC state.
 889  */
 890 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
 891 {
 892         u32 ioc_status, ioc_config;
 893         u8 ready, enabled;
 894
 895         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 896         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 897
 898         if (mrioc->unrecoverable)
 899                 return MRIOC_STATE_UNRECOVERABLE;
 900         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
 901                 return MRIOC_STATE_FAULT;
 902
 903         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
 904         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
 905
 906         if (ready && enabled)
 907                 return MRIOC_STATE_READY;
 908         if ((!ready) && (!enabled))
 909                 return MRIOC_STATE_RESET;
 910         if ((!ready) && (enabled))
 911                 return MRIOC_STATE_BECOMING_READY;
 912
 913         return MRIOC_STATE_RESET_REQUESTED;
 914 }
 915
 916 /**
 917  * mpi3mr_clear_reset_history - clear reset history
 918  * @mrioc: Adapter instance reference
 919  *
 920  * Write the reset history bit in IOC status to clear the bit,
 921  * if it is already set.
 922  *
 923  * Return: Nothing.
 924  */
 925 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
 926 {
 927         u32 ioc_status;
 928
 929         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 930         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
 931                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
 932 }
 933
 934 /**
 935  * mpi3mr_issue_and_process_mur - Message unit Reset handler
 936  * @mrioc: Adapter instance reference
 937  * @reset_reason: Reset reason code
 938  *
 939  * Issue Message unit Reset to the controller and wait for it to
 940  * be complete.
 941  *
 942  * Return: 0 on success, -1 on failure.
 943  */
 944 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
 945         u32 reset_reason)
 946 {
 947         u32 ioc_config, timeout, ioc_status;
 948         int retval = -1;
 949
 950         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
 951         if (mrioc->unrecoverable) {
 952                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
 953                 return retval;
 954         }
 955         mpi3mr_clear_reset_history(mrioc);
 956         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
 957         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 958         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
 959         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 960
 961         timeout = mrioc->ready_timeout * 10;
 962         do {
 963                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 964                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
 965                         mpi3mr_clear_reset_history(mrioc);
 966                         ioc_config =
 967                             readl(&mrioc->sysif_regs->ioc_configuration);
 968                         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
 969                               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
 970                             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC))) {
 971                                 retval = 0;
 972                                 break;
 973                         }
 974                 }
 975                 msleep(100);
 976         } while (--timeout);
 977
 978         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 979         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 980
 981         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
 982             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
 983         return retval;
 984 }
 985
 986 /**
 987  * mpi3mr_bring_ioc_ready - Bring controller to ready state
 988  * @mrioc: Adapter instance reference
 989  *
 990  * Set Enable IOC bit in IOC configuration register and wait for
 991  * the controller to become ready.
 992  *
 993  * Return: 0 on success, -1 on failure.
 994  */
 995 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
 996 {
 997         u32 ioc_config, timeout;
 998         enum mpi3mr_iocstate current_state;
 999
1000         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1001         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1002         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1003
1004         timeout = mrioc->ready_timeout * 10;
1005         do {
1006                 current_state = mpi3mr_get_iocstate(mrioc);
1007                 if (current_state == MRIOC_STATE_READY)
1008                         return 0;
1009                 msleep(100);
1010         } while (--timeout);
1011
1012         return -1;
1013 }
1014
1015 /**
1016  * mpi3mr_soft_reset_success - Check softreset is success or not
1017  * @ioc_status: IOC status register value
1018  * @ioc_config: IOC config register value
1019  *
1020  * Check whether the soft reset is successful or not based on
1021  * IOC status and IOC config register values.
1022  *
1023  * Return: True when the soft reset is success, false otherwise.
1024  */
1025 static inline bool
1026 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1027 {
1028         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1029             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1030             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1031                 return true;
1032         return false;
1033 }
1034
1035 /**
1036  * mpi3mr_diagfault_success - Check diag fault is success or not
1037  * @mrioc: Adapter reference
1038  * @ioc_status: IOC status register value
1039  *
1040  * Check whether the controller hit diag reset fault code.
1041  *
1042  * Return: True when there is diag fault, false otherwise.
1043  */
1044 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1045         u32 ioc_status)
1046 {
1047         u32 fault;
1048
1049         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1050                 return false;
1051         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1052         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET)
1053                 return true;
1054         return false;
1055 }
1056
1057 /**
1058  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1059  * @mrioc: Adapter reference
1060  *
1061  * Set diag save bit in IOC configuration register to enable
1062  * snapdump.
1063  *
1064  * Return: Nothing.
1065  */
1066 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1067 {
1068         u32 ioc_config;
1069
1070         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1071         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1072         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1073 }
1074
1075 /**
1076  * mpi3mr_issue_reset - Issue reset to the controller
1077  * @mrioc: Adapter reference
1078  * @reset_type: Reset type
1079  * @reset_reason: Reset reason code
1080  *
1081  * Unlock the host diagnostic registers and write the specific
1082  * reset type to that, wait for reset acknowledgment from the
1083  * controller, if the reset is not successful retry for the
1084  * predefined number of times.
1085  *
1086  * Return: 0 on success, non-zero on failure.
1087  */
1088 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1089         u32 reset_reason)
1090 {
1091         int retval = -1;
1092         u8 unlock_retry_count, reset_retry_count = 0;
1093         u32 host_diagnostic, timeout, ioc_status, ioc_config;
1094
1095         pci_cfg_access_lock(mrioc->pdev);
1096         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1097             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1098                 goto out;
1099         if (mrioc->unrecoverable)
1100                 goto out;
1101 retry_reset:
1102         unlock_retry_count = 0;
1103         mpi3mr_clear_reset_history(mrioc);
1104         do {
1105                 ioc_info(mrioc,
1106                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1107                     ++unlock_retry_count);
1108                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1109                         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1110                         mrioc->unrecoverable = 1;
1111                         goto out;
1112                 }
1113
1114                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1115                     &mrioc->sysif_regs->write_sequence);
1116                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1117                     &mrioc->sysif_regs->write_sequence);
1118                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1119                     &mrioc->sysif_regs->write_sequence);
1120                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1121                     &mrioc->sysif_regs->write_sequence);
1122                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1123                     &mrioc->sysif_regs->write_sequence);
1124                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1125                     &mrioc->sysif_regs->write_sequence);
1126                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1127                     &mrioc->sysif_regs->write_sequence);
1128                 usleep_range(1000, 1100);
1129                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1130                 ioc_info(mrioc,
1131                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1132                     unlock_retry_count, host_diagnostic);
1133         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1134
1135         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1136         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1137             mpi3mr_reset_type_name(reset_type),
1138             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1139         writel(host_diagnostic | reset_type,
1140             &mrioc->sysif_regs->host_diagnostic);
1141         timeout = mrioc->ready_timeout * 10;
1142         if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) {
1143                 do {
1144                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1145                         if (ioc_status &
1146                             MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
1147                                 mpi3mr_clear_reset_history(mrioc);
1148                                 ioc_config =
1149                                     readl(&mrioc->sysif_regs->ioc_configuration);
1150                                 if (mpi3mr_soft_reset_success(ioc_status,
1151                                     ioc_config)) {
1152                                         retval = 0;
1153                                         break;
1154                                 }
1155                         }
1156                         msleep(100);
1157                 } while (--timeout);
1158                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1159                     &mrioc->sysif_regs->write_sequence);
1160         } else if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT) {
1161                 do {
1162                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1163                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1164                                 retval = 0;
1165                                 break;
1166                         }
1167                         msleep(100);
1168                 } while (--timeout);
1169                 mpi3mr_clear_reset_history(mrioc);
1170                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1171                     &mrioc->sysif_regs->write_sequence);
1172         }
1173         if (retval && ((++reset_retry_count) < MPI3MR_MAX_RESET_RETRY_COUNT)) {
1174                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1175                 ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1176                 ioc_info(mrioc,
1177                     "Base IOC Sts/Config after reset try %d is (0x%x)/(0x%x)\n",
1178                     reset_retry_count, ioc_status, ioc_config);
1179                 goto retry_reset;
1180         }
1181
1182 out:
1183         pci_cfg_access_unlock(mrioc->pdev);
1184         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1185         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1186
1187         ioc_info(mrioc,
1188             "Base IOC Sts/Config after %s reset is (0x%x)/(0x%x)\n",
1189             (!retval) ? "successful" : "failed", ioc_status,
1190             ioc_config);
1191         return retval;
1192 }
1193
1194 /**
1195  * mpi3mr_admin_request_post - Post request to admin queue
1196  * @mrioc: Adapter reference
1197  * @admin_req: MPI3 request
1198  * @admin_req_sz: Request size
1199  * @ignore_reset: Ignore reset in process
1200  *
1201  * Post the MPI3 request into admin request queue and
1202  * inform the controller, if the queue is full return
1203  * appropriate error.
1204  *
1205  * Return: 0 on success, non-zero on failure.
1206  */
1207 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1208         u16 admin_req_sz, u8 ignore_reset)
1209 {
1210         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1211         int retval = 0;
1212         unsigned long flags;
1213         u8 *areq_entry;
1214
1215         if (mrioc->unrecoverable) {
1216                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1217                 return -EFAULT;
1218         }
1219
1220         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1221         areq_pi = mrioc->admin_req_pi;
1222         areq_ci = mrioc->admin_req_ci;
1223         max_entries = mrioc->num_admin_req;
1224         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1225             (areq_pi == (max_entries - 1)))) {
1226                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1227                 retval = -EAGAIN;
1228                 goto out;
1229         }
1230         if (!ignore_reset && mrioc->reset_in_progress) {
1231                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1232                 retval = -EAGAIN;
1233                 goto out;
1234         }
1235         areq_entry = (u8 *)mrioc->admin_req_base +
1236             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1237         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1238         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1239
1240         if (++areq_pi == max_entries)
1241                 areq_pi = 0;
1242         mrioc->admin_req_pi = areq_pi;
1243
1244         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1245
1246 out:
1247         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1248
1249         return retval;
1250 }
1251
1252 /**
1253  * mpi3mr_free_op_req_q_segments - free request memory segments
1254  * @mrioc: Adapter instance reference
1255  * @q_idx: operational request queue index
1256  *
1257  * Free memory segments allocated for operational request queue
1258  *
1259  * Return: Nothing.
1260  */
1261 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1262 {
1263         u16 j;
1264         int size;
1265         struct segments *segments;
1266
1267         segments = mrioc->req_qinfo[q_idx].q_segments;
1268         if (!segments)
1269                 return;
1270
1271         if (mrioc->enable_segqueue) {
1272                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1273                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1274                         dma_free_coherent(&mrioc->pdev->dev,
1275                             MPI3MR_MAX_SEG_LIST_SIZE,
1276                             mrioc->req_qinfo[q_idx].q_segment_list,
1277                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1278                         mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1279                 }
1280         } else
1281                 size = mrioc->req_qinfo[q_idx].segment_qd *
1282                     mrioc->facts.op_req_sz;
1283
1284         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1285                 if (!segments[j].segment)
1286                         continue;
1287                 dma_free_coherent(&mrioc->pdev->dev,
1288                     size, segments[j].segment, segments[j].segment_dma);
1289                 segments[j].segment = NULL;
1290         }
1291         kfree(mrioc->req_qinfo[q_idx].q_segments);
1292         mrioc->req_qinfo[q_idx].q_segments = NULL;
1293         mrioc->req_qinfo[q_idx].qid = 0;
1294 }
1295
1296 /**
1297  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1298  * @mrioc: Adapter instance reference
1299  * @q_idx: operational reply queue index
1300  *
1301  * Free memory segments allocated for operational reply queue
1302  *
1303  * Return: Nothing.
1304  */
1305 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1306 {
1307         u16 j;
1308         int size;
1309         struct segments *segments;
1310
1311         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1312         if (!segments)
1313                 return;
1314
1315         if (mrioc->enable_segqueue) {
1316                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1317                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1318                         dma_free_coherent(&mrioc->pdev->dev,
1319                             MPI3MR_MAX_SEG_LIST_SIZE,
1320                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1321                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1322                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1323                 }
1324         } else
1325                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1326                     mrioc->op_reply_desc_sz;
1327
1328         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1329                 if (!segments[j].segment)
1330                         continue;
1331                 dma_free_coherent(&mrioc->pdev->dev,
1332                     size, segments[j].segment, segments[j].segment_dma);
1333                 segments[j].segment = NULL;
1334         }
1335
1336         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1337         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1338         mrioc->op_reply_qinfo[q_idx].qid = 0;
1339 }
1340
1341 /**
1342  * mpi3mr_delete_op_reply_q - delete operational reply queue
1343  * @mrioc: Adapter instance reference
1344  * @qidx: operational reply queue index
1345  *
1346  * Delete operatinal reply queue by issuing MPI request
1347  * through admin queue.
1348  *
1349  * Return:  0 on success, non-zero on failure.
1350  */
1351 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1352 {
1353         struct mpi3_delete_reply_queue_request delq_req;
1354         int retval = 0;
1355         u16 reply_qid = 0, midx;
1356
1357         reply_qid = mrioc->op_reply_qinfo[qidx].qid;
1358
1359         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1360
1361         if (!reply_qid) {
1362                 retval = -1;
1363                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1364                 goto out;
1365         }
1366
1367         memset(&delq_req, 0, sizeof(delq_req));
1368         mutex_lock(&mrioc->init_cmds.mutex);
1369         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1370                 retval = -1;
1371                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1372                 mutex_unlock(&mrioc->init_cmds.mutex);
1373                 goto out;
1374         }
1375         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1376         mrioc->init_cmds.is_waiting = 1;
1377         mrioc->init_cmds.callback = NULL;
1378         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1379         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1380         delq_req.queue_id = cpu_to_le16(reply_qid);
1381
1382         init_completion(&mrioc->init_cmds.done);
1383         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1384             1);
1385         if (retval) {
1386                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1387                 goto out_unlock;
1388         }
1389         wait_for_completion_timeout(&mrioc->init_cmds.done,
1390             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1391         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1392                 ioc_err(mrioc, "Issue DelRepQ: command timed out\n");
1393                 mpi3mr_set_diagsave(mrioc);
1394                 mpi3mr_issue_reset(mrioc,
1395                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1396                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1397                 mrioc->unrecoverable = 1;
1398
1399                 retval = -1;
1400                 goto out_unlock;
1401         }
1402         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1403             != MPI3_IOCSTATUS_SUCCESS) {
1404                 ioc_err(mrioc,
1405                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1406                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1407                     mrioc->init_cmds.ioc_loginfo);
1408                 retval = -1;
1409                 goto out_unlock;
1410         }
1411         mrioc->intr_info[midx].op_reply_q = NULL;
1412
1413         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1414 out_unlock:
1415         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1416         mutex_unlock(&mrioc->init_cmds.mutex);
1417 out:
1418
1419         return retval;
1420 }
1421
1422 /**
1423  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1424  * @mrioc: Adapter instance reference
1425  * @qidx: request queue index
1426  *
1427  * Allocate segmented memory pools for operational reply
1428  * queue.
1429  *
1430  * Return: 0 on success, non-zero on failure.
1431  */
1432 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1433 {
1434         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1435         int i, size;
1436         u64 *q_segment_list_entry = NULL;
1437         struct segments *segments;
1438
1439         if (mrioc->enable_segqueue) {
1440                 op_reply_q->segment_qd =
1441                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1442
1443                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1444
1445                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1446                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1447                     GFP_KERNEL);
1448                 if (!op_reply_q->q_segment_list)
1449                         return -ENOMEM;
1450                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1451         } else {
1452                 op_reply_q->segment_qd = op_reply_q->num_replies;
1453                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1454         }
1455
1456         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1457             op_reply_q->segment_qd);
1458
1459         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1460             sizeof(struct segments), GFP_KERNEL);
1461         if (!op_reply_q->q_segments)
1462                 return -ENOMEM;
1463
1464         segments = op_reply_q->q_segments;
1465         for (i = 0; i < op_reply_q->num_segments; i++) {
1466                 segments[i].segment =
1467                     dma_alloc_coherent(&mrioc->pdev->dev,
1468                     size, &segments[i].segment_dma, GFP_KERNEL);
1469                 if (!segments[i].segment)
1470                         return -ENOMEM;
1471                 if (mrioc->enable_segqueue)
1472                         q_segment_list_entry[i] =
1473                             (unsigned long)segments[i].segment_dma;
1474         }
1475
1476         return 0;
1477 }
1478
1479 /**
1480  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1481  * @mrioc: Adapter instance reference
1482  * @qidx: request queue index
1483  *
1484  * Allocate segmented memory pools for operational request
1485  * queue.
1486  *
1487  * Return: 0 on success, non-zero on failure.
1488  */
1489 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1490 {
1491         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1492         int i, size;
1493         u64 *q_segment_list_entry = NULL;
1494         struct segments *segments;
1495
1496         if (mrioc->enable_segqueue) {
1497                 op_req_q->segment_qd =
1498                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1499
1500                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1501
1502                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1503                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1504                     GFP_KERNEL);
1505                 if (!op_req_q->q_segment_list)
1506                         return -ENOMEM;
1507                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1508
1509         } else {
1510                 op_req_q->segment_qd = op_req_q->num_requests;
1511                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1512         }
1513
1514         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1515             op_req_q->segment_qd);
1516
1517         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1518             sizeof(struct segments), GFP_KERNEL);
1519         if (!op_req_q->q_segments)
1520                 return -ENOMEM;
1521
1522         segments = op_req_q->q_segments;
1523         for (i = 0; i < op_req_q->num_segments; i++) {
1524                 segments[i].segment =
1525                     dma_alloc_coherent(&mrioc->pdev->dev,
1526                     size, &segments[i].segment_dma, GFP_KERNEL);
1527                 if (!segments[i].segment)
1528                         return -ENOMEM;
1529                 if (mrioc->enable_segqueue)
1530                         q_segment_list_entry[i] =
1531                             (unsigned long)segments[i].segment_dma;
1532         }
1533
1534         return 0;
1535 }
1536
1537 /**
1538  * mpi3mr_create_op_reply_q - create operational reply queue
1539  * @mrioc: Adapter instance reference
1540  * @qidx: operational reply queue index
1541  *
1542  * Create operatinal reply queue by issuing MPI request
1543  * through admin queue.
1544  *
1545  * Return:  0 on success, non-zero on failure.
1546  */
1547 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1548 {
1549         struct mpi3_create_reply_queue_request create_req;
1550         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1551         int retval = 0;
1552         u16 reply_qid = 0, midx;
1553
1554         reply_qid = op_reply_q->qid;
1555
1556         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1557
1558         if (reply_qid) {
1559                 retval = -1;
1560                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1561                     reply_qid);
1562
1563                 return retval;
1564         }
1565
1566         reply_qid = qidx + 1;
1567         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1568         if (!mrioc->pdev->revision)
1569                 op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
1570         op_reply_q->ci = 0;
1571         op_reply_q->ephase = 1;
1572         atomic_set(&op_reply_q->pend_ios, 0);
1573         atomic_set(&op_reply_q->in_use, 0);
1574         op_reply_q->enable_irq_poll = false;
1575
1576         if (!op_reply_q->q_segments) {
1577                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1578                 if (retval) {
1579                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1580                         goto out;
1581                 }
1582         }
1583
1584         memset(&create_req, 0, sizeof(create_req));
1585         mutex_lock(&mrioc->init_cmds.mutex);
1586         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1587                 retval = -1;
1588                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1589                 goto out_unlock;
1590         }
1591         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1592         mrioc->init_cmds.is_waiting = 1;
1593         mrioc->init_cmds.callback = NULL;
1594         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1595         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1596         create_req.queue_id = cpu_to_le16(reply_qid);
1597         create_req.flags = MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1598         create_req.msix_index = cpu_to_le16(mrioc->intr_info[midx].msix_index);
1599         if (mrioc->enable_segqueue) {
1600                 create_req.flags |=
1601                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1602                 create_req.base_address = cpu_to_le64(
1603                     op_reply_q->q_segment_list_dma);
1604         } else
1605                 create_req.base_address = cpu_to_le64(
1606                     op_reply_q->q_segments[0].segment_dma);
1607
1608         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1609
1610         init_completion(&mrioc->init_cmds.done);
1611         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1612             sizeof(create_req), 1);
1613         if (retval) {
1614                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1615                 goto out_unlock;
1616         }
1617         wait_for_completion_timeout(&mrioc->init_cmds.done,
1618             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1619         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1620                 ioc_err(mrioc, "CreateRepQ: command timed out\n");
1621                 mpi3mr_set_diagsave(mrioc);
1622                 mpi3mr_issue_reset(mrioc,
1623                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1624                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1625                 mrioc->unrecoverable = 1;
1626                 retval = -1;
1627                 goto out_unlock;
1628         }
1629         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1630             != MPI3_IOCSTATUS_SUCCESS) {
1631                 ioc_err(mrioc,
1632                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1633                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1634                     mrioc->init_cmds.ioc_loginfo);
1635                 retval = -1;
1636                 goto out_unlock;
1637         }
1638         op_reply_q->qid = reply_qid;
1639         mrioc->intr_info[midx].op_reply_q = op_reply_q;
1640
1641 out_unlock:
1642         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1643         mutex_unlock(&mrioc->init_cmds.mutex);
1644 out:
1645
1646         return retval;
1647 }
1648
1649 /**
1650  * mpi3mr_create_op_req_q - create operational request queue
1651  * @mrioc: Adapter instance reference
1652  * @idx: operational request queue index
1653  * @reply_qid: Reply queue ID
1654  *
1655  * Create operatinal request queue by issuing MPI request
1656  * through admin queue.
1657  *
1658  * Return:  0 on success, non-zero on failure.
1659  */
1660 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1661         u16 reply_qid)
1662 {
1663         struct mpi3_create_request_queue_request create_req;
1664         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1665         int retval = 0;
1666         u16 req_qid = 0;
1667
1668         req_qid = op_req_q->qid;
1669
1670         if (req_qid) {
1671                 retval = -1;
1672                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1673                     req_qid);
1674
1675                 return retval;
1676         }
1677         req_qid = idx + 1;
1678
1679         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1680         op_req_q->ci = 0;
1681         op_req_q->pi = 0;
1682         op_req_q->reply_qid = reply_qid;
1683         spin_lock_init(&op_req_q->q_lock);
1684
1685         if (!op_req_q->q_segments) {
1686                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1687                 if (retval) {
1688                         mpi3mr_free_op_req_q_segments(mrioc, idx);
1689                         goto out;
1690                 }
1691         }
1692
1693         memset(&create_req, 0, sizeof(create_req));
1694         mutex_lock(&mrioc->init_cmds.mutex);
1695         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1696                 retval = -1;
1697                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
1698                 goto out_unlock;
1699         }
1700         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1701         mrioc->init_cmds.is_waiting = 1;
1702         mrioc->init_cmds.callback = NULL;
1703         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1704         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
1705         create_req.queue_id = cpu_to_le16(req_qid);
1706         if (mrioc->enable_segqueue) {
1707                 create_req.flags =
1708                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1709                 create_req.base_address = cpu_to_le64(
1710                     op_req_q->q_segment_list_dma);
1711         } else
1712                 create_req.base_address = cpu_to_le64(
1713                     op_req_q->q_segments[0].segment_dma);
1714         create_req.reply_queue_id = cpu_to_le16(reply_qid);
1715         create_req.size = cpu_to_le16(op_req_q->num_requests);
1716
1717         init_completion(&mrioc->init_cmds.done);
1718         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1719             sizeof(create_req), 1);
1720         if (retval) {
1721                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
1722                 goto out_unlock;
1723         }
1724         wait_for_completion_timeout(&mrioc->init_cmds.done,
1725             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1726         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1727                 ioc_err(mrioc, "CreateReqQ: command timed out\n");
1728                 mpi3mr_set_diagsave(mrioc);
1729                 if (mpi3mr_issue_reset(mrioc,
1730                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1731                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT))
1732                         mrioc->unrecoverable = 1;
1733                 retval = -1;
1734                 goto out_unlock;
1735         }
1736         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1737             != MPI3_IOCSTATUS_SUCCESS) {
1738                 ioc_err(mrioc,
1739                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1740                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1741                     mrioc->init_cmds.ioc_loginfo);
1742                 retval = -1;
1743                 goto out_unlock;
1744         }
1745         op_req_q->qid = req_qid;
1746
1747 out_unlock:
1748         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1749         mutex_unlock(&mrioc->init_cmds.mutex);
1750 out:
1751
1752         return retval;
1753 }
1754
1755 /**
1756  * mpi3mr_create_op_queues - create operational queue pairs
1757  * @mrioc: Adapter instance reference
1758  *
1759  * Allocate memory for operational queue meta data and call
1760  * create request and reply queue functions.
1761  *
1762  * Return: 0 on success, non-zero on failures.
1763  */
1764 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
1765 {
1766         int retval = 0;
1767         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
1768
1769         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
1770             mrioc->facts.max_op_req_q);
1771
1772         msix_count_op_q =
1773             mrioc->intr_info_count - mrioc->op_reply_q_offset;
1774         if (!mrioc->num_queues)
1775                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
1776         num_queues = mrioc->num_queues;
1777         ioc_info(mrioc, "Trying to create %d Operational Q pairs\n",
1778             num_queues);
1779
1780         if (!mrioc->req_qinfo) {
1781                 mrioc->req_qinfo = kcalloc(num_queues,
1782                     sizeof(struct op_req_qinfo), GFP_KERNEL);
1783                 if (!mrioc->req_qinfo) {
1784                         retval = -1;
1785                         goto out_failed;
1786                 }
1787
1788                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
1789                     num_queues, GFP_KERNEL);
1790                 if (!mrioc->op_reply_qinfo) {
1791                         retval = -1;
1792                         goto out_failed;
1793                 }
1794         }
1795
1796         if (mrioc->enable_segqueue)
1797                 ioc_info(mrioc,
1798                     "allocating operational queues through segmented queues\n");
1799
1800         for (i = 0; i < num_queues; i++) {
1801                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
1802                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
1803                         break;
1804                 }
1805                 if (mpi3mr_create_op_req_q(mrioc, i,
1806                     mrioc->op_reply_qinfo[i].qid)) {
1807                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
1808                         mpi3mr_delete_op_reply_q(mrioc, i);
1809                         break;
1810                 }
1811         }
1812
1813         if (i == 0) {
1814                 /* Not even one queue is created successfully*/
1815                 retval = -1;
1816                 goto out_failed;
1817         }
1818         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
1819         ioc_info(mrioc, "Successfully created %d Operational Q pairs\n",
1820             mrioc->num_op_reply_q);
1821
1822         return retval;
1823 out_failed:
1824         kfree(mrioc->req_qinfo);
1825         mrioc->req_qinfo = NULL;
1826
1827         kfree(mrioc->op_reply_qinfo);
1828         mrioc->op_reply_qinfo = NULL;
1829
1830         return retval;
1831 }
1832
1833 /**
1834  * mpi3mr_op_request_post - Post request to operational queue
1835  * @mrioc: Adapter reference
1836  * @op_req_q: Operational request queue info
1837  * @req: MPI3 request
1838  *
1839  * Post the MPI3 request into operational request queue and
1840  * inform the controller, if the queue is full return
1841  * appropriate error.
1842  *
1843  * Return: 0 on success, non-zero on failure.
1844  */
1845 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
1846         struct op_req_qinfo *op_req_q, u8 *req)
1847 {
1848         u16 pi = 0, max_entries, reply_qidx = 0, midx;
1849         int retval = 0;
1850         unsigned long flags;
1851         u8 *req_entry;
1852         void *segment_base_addr;
1853         u16 req_sz = mrioc->facts.op_req_sz;
1854         struct segments *segments = op_req_q->q_segments;
1855
1856         reply_qidx = op_req_q->reply_qid - 1;
1857
1858         if (mrioc->unrecoverable)
1859                 return -EFAULT;
1860
1861         spin_lock_irqsave(&op_req_q->q_lock, flags);
1862         pi = op_req_q->pi;
1863         max_entries = op_req_q->num_requests;
1864
1865         if (mpi3mr_check_req_qfull(op_req_q)) {
1866                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
1867                     reply_qidx, mrioc->op_reply_q_offset);
1868                 mpi3mr_process_op_reply_q(mrioc, &mrioc->intr_info[midx]);
1869
1870                 if (mpi3mr_check_req_qfull(op_req_q)) {
1871                         retval = -EAGAIN;
1872                         goto out;
1873                 }
1874         }
1875
1876         if (mrioc->reset_in_progress) {
1877                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
1878                 retval = -EAGAIN;
1879                 goto out;
1880         }
1881
1882         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
1883         req_entry = (u8 *)segment_base_addr +
1884             ((pi % op_req_q->segment_qd) * req_sz);
1885
1886         memset(req_entry, 0, req_sz);
1887         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
1888
1889         if (++pi == max_entries)
1890                 pi = 0;
1891         op_req_q->pi = pi;
1892
1893         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
1894             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
1895                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
1896
1897         writel(op_req_q->pi,
1898             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
1899
1900 out:
1901         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
1902         return retval;
1903 }
1904
1905 /**
1906  * mpi3mr_sync_timestamp - Issue time stamp sync request
1907  * @mrioc: Adapter reference
1908  *
1909  * Issue IO unit control MPI request to synchornize firmware
1910  * timestamp with host time.
1911  *
1912  * Return: 0 on success, non-zero on failure.
1913  */
1914 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
1915 {
1916         ktime_t current_time;
1917         struct mpi3_iounit_control_request iou_ctrl;
1918         int retval = 0;
1919
1920         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
1921         mutex_lock(&mrioc->init_cmds.mutex);
1922         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1923                 retval = -1;
1924                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
1925                 mutex_unlock(&mrioc->init_cmds.mutex);
1926                 goto out;
1927         }
1928         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1929         mrioc->init_cmds.is_waiting = 1;
1930         mrioc->init_cmds.callback = NULL;
1931         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1932         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
1933         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
1934         current_time = ktime_get_real();
1935         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
1936
1937         init_completion(&mrioc->init_cmds.done);
1938         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
1939             sizeof(iou_ctrl), 0);
1940         if (retval) {
1941                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
1942                 goto out_unlock;
1943         }
1944
1945         wait_for_completion_timeout(&mrioc->init_cmds.done,
1946             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1947         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1948                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
1949                 mrioc->init_cmds.is_waiting = 0;
1950                 mpi3mr_soft_reset_handler(mrioc,
1951                     MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
1952                 retval = -1;
1953                 goto out_unlock;
1954         }
1955         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1956             != MPI3_IOCSTATUS_SUCCESS) {
1957                 ioc_err(mrioc,
1958                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1959                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1960                     mrioc->init_cmds.ioc_loginfo);
1961                 retval = -1;
1962                 goto out_unlock;
1963         }
1964
1965 out_unlock:
1966         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1967         mutex_unlock(&mrioc->init_cmds.mutex);
1968
1969 out:
1970         return retval;
1971 }
1972
1973 /**
1974  * mpi3mr_watchdog_work - watchdog thread to monitor faults
1975  * @work: work struct
1976  *
1977  * Watch dog work periodically executed (1 second interval) to
1978  * monitor firmware fault and to issue periodic timer sync to
1979  * the firmware.
1980  *
1981  * Return: Nothing.
1982  */
1983 static void mpi3mr_watchdog_work(struct work_struct *work)
1984 {
1985         struct mpi3mr_ioc *mrioc =
1986             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
1987         unsigned long flags;
1988         enum mpi3mr_iocstate ioc_state;
1989         u32 fault, host_diagnostic;
1990
1991         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
1992                 mrioc->ts_update_counter = 0;
1993                 mpi3mr_sync_timestamp(mrioc);
1994         }
1995
1996         /*Check for fault state every one second and issue Soft reset*/
1997         ioc_state = mpi3mr_get_iocstate(mrioc);
1998         if (ioc_state == MRIOC_STATE_FAULT) {
1999                 fault = readl(&mrioc->sysif_regs->fault) &
2000                     MPI3_SYSIF_FAULT_CODE_MASK;
2001                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2002                 if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2003                         if (!mrioc->diagsave_timeout) {
2004                                 mpi3mr_print_fault_info(mrioc);
2005                                 ioc_warn(mrioc, "Diag save in progress\n");
2006                         }
2007                         if ((mrioc->diagsave_timeout++) <=
2008                             MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2009                                 goto schedule_work;
2010                 } else
2011                         mpi3mr_print_fault_info(mrioc);
2012                 mrioc->diagsave_timeout = 0;
2013
2014                 if (fault == MPI3_SYSIF_FAULT_CODE_FACTORY_RESET) {
2015                         ioc_info(mrioc,
2016                             "Factory Reset fault occurred marking controller as unrecoverable"
2017                             );
2018                         mrioc->unrecoverable = 1;
2019                         goto out;
2020                 }
2021
2022                 if ((fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) ||
2023                     (fault == MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS) ||
2024                     (mrioc->reset_in_progress))
2025                         goto out;
2026                 if (fault == MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET)
2027                         mpi3mr_soft_reset_handler(mrioc,
2028                             MPI3MR_RESET_FROM_CIACTIV_FAULT, 0);
2029                 else
2030                         mpi3mr_soft_reset_handler(mrioc,
2031                             MPI3MR_RESET_FROM_FAULT_WATCH, 0);
2032         }
2033
2034 schedule_work:
2035         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2036         if (mrioc->watchdog_work_q)
2037                 queue_delayed_work(mrioc->watchdog_work_q,
2038                     &mrioc->watchdog_work,
2039                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2040         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2041 out:
2042         return;
2043 }
2044
2045 /**
2046  * mpi3mr_start_watchdog - Start watchdog
2047  * @mrioc: Adapter instance reference
2048  *
2049  * Create and start the watchdog thread to monitor controller
2050  * faults.
2051  *
2052  * Return: Nothing.
2053  */
2054 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2055 {
2056         if (mrioc->watchdog_work_q)
2057                 return;
2058
2059         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2060         snprintf(mrioc->watchdog_work_q_name,
2061             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2062             mrioc->id);
2063         mrioc->watchdog_work_q =
2064             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2065         if (!mrioc->watchdog_work_q) {
2066                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2067                 return;
2068         }
2069
2070         if (mrioc->watchdog_work_q)
2071                 queue_delayed_work(mrioc->watchdog_work_q,
2072                     &mrioc->watchdog_work,
2073                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2074 }
2075
2076 /**
2077  * mpi3mr_stop_watchdog - Stop watchdog
2078  * @mrioc: Adapter instance reference
2079  *
2080  * Stop the watchdog thread created to monitor controller
2081  * faults.
2082  *
2083  * Return: Nothing.
2084  */
2085 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2086 {
2087         unsigned long flags;
2088         struct workqueue_struct *wq;
2089
2090         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2091         wq = mrioc->watchdog_work_q;
2092         mrioc->watchdog_work_q = NULL;
2093         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2094         if (wq) {
2095                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2096                         flush_workqueue(wq);
2097                 destroy_workqueue(wq);
2098         }
2099 }
2100
2101 /**
2102  * mpi3mr_kill_ioc - Kill the controller
2103  * @mrioc: Adapter instance reference
2104  * @reason: reason for the failure.
2105  *
2106  * If fault debug is enabled, display the fault info else issue
2107  * diag fault and freeze the system for controller debug
2108  * purpose.
2109  *
2110  * Return: Nothing.
2111  */
2112 static void mpi3mr_kill_ioc(struct mpi3mr_ioc *mrioc, u32 reason)
2113 {
2114         enum mpi3mr_iocstate ioc_state;
2115
2116         if (!mrioc->fault_dbg)
2117                 return;
2118
2119         dump_stack();
2120
2121         ioc_state = mpi3mr_get_iocstate(mrioc);
2122         if (ioc_state == MRIOC_STATE_FAULT)
2123                 mpi3mr_print_fault_info(mrioc);
2124         else {
2125                 ioc_err(mrioc, "Firmware is halted due to the reason %d\n",
2126                     reason);
2127                 mpi3mr_diagfault_reset_handler(mrioc, reason);
2128         }
2129         if (mrioc->fault_dbg == 2)
2130                 for (;;)
2131                         ;
2132         else
2133                 panic("panic in %s\n", __func__);
2134 }
2135
2136 /**
2137  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2138  * @mrioc: Adapter instance reference
2139  *
2140  * Allocate memory for admin queue pair if required and register
2141  * the admin queue with the controller.
2142  *
2143  * Return: 0 on success, non-zero on failures.
2144  */
2145 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2146 {
2147         int retval = 0;
2148         u32 num_admin_entries = 0;
2149
2150         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2151         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2152             MPI3MR_ADMIN_REQ_FRAME_SZ;
2153         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2154         mrioc->admin_req_base = NULL;
2155
2156         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2157         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2158             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2159         mrioc->admin_reply_ci = 0;
2160         mrioc->admin_reply_ephase = 1;
2161         mrioc->admin_reply_base = NULL;
2162
2163         if (!mrioc->admin_req_base) {
2164                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2165                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2166
2167                 if (!mrioc->admin_req_base) {
2168                         retval = -1;
2169                         goto out_failed;
2170                 }
2171
2172                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2173                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2174                     GFP_KERNEL);
2175
2176                 if (!mrioc->admin_reply_base) {
2177                         retval = -1;
2178                         goto out_failed;
2179                 }
2180         }
2181
2182         num_admin_entries = (mrioc->num_admin_replies << 16) |
2183             (mrioc->num_admin_req);
2184         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2185         mpi3mr_writeq(mrioc->admin_req_dma,
2186             &mrioc->sysif_regs->admin_request_queue_address);
2187         mpi3mr_writeq(mrioc->admin_reply_dma,
2188             &mrioc->sysif_regs->admin_reply_queue_address);
2189         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2190         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2191         return retval;
2192
2193 out_failed:
2194
2195         if (mrioc->admin_reply_base) {
2196                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2197                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2198                 mrioc->admin_reply_base = NULL;
2199         }
2200         if (mrioc->admin_req_base) {
2201                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2202                     mrioc->admin_req_base, mrioc->admin_req_dma);
2203                 mrioc->admin_req_base = NULL;
2204         }
2205         return retval;
2206 }
2207
2208 /**
2209  * mpi3mr_issue_iocfacts - Send IOC Facts
2210  * @mrioc: Adapter instance reference
2211  * @facts_data: Cached IOC facts data
2212  *
2213  * Issue IOC Facts MPI request through admin queue and wait for
2214  * the completion of it or time out.
2215  *
2216  * Return: 0 on success, non-zero on failures.
2217  */
2218 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2219         struct mpi3_ioc_facts_data *facts_data)
2220 {
2221         struct mpi3_ioc_facts_request iocfacts_req;
2222         void *data = NULL;
2223         dma_addr_t data_dma;
2224         u32 data_len = sizeof(*facts_data);
2225         int retval = 0;
2226         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2227
2228         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2229             GFP_KERNEL);
2230
2231         if (!data) {
2232                 retval = -1;
2233                 goto out;
2234         }
2235
2236         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2237         mutex_lock(&mrioc->init_cmds.mutex);
2238         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2239                 retval = -1;
2240                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2241                 mutex_unlock(&mrioc->init_cmds.mutex);
2242                 goto out;
2243         }
2244         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2245         mrioc->init_cmds.is_waiting = 1;
2246         mrioc->init_cmds.callback = NULL;
2247         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2248         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2249
2250         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2251             data_dma);
2252
2253         init_completion(&mrioc->init_cmds.done);
2254         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2255             sizeof(iocfacts_req), 1);
2256         if (retval) {
2257                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2258                 goto out_unlock;
2259         }
2260         wait_for_completion_timeout(&mrioc->init_cmds.done,
2261             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2262         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2263                 ioc_err(mrioc, "Issue IOCFacts: command timed out\n");
2264                 mpi3mr_set_diagsave(mrioc);
2265                 mpi3mr_issue_reset(mrioc,
2266                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2267                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2268                 mrioc->unrecoverable = 1;
2269                 retval = -1;
2270                 goto out_unlock;
2271         }
2272         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2273             != MPI3_IOCSTATUS_SUCCESS) {
2274                 ioc_err(mrioc,
2275                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2276                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2277                     mrioc->init_cmds.ioc_loginfo);
2278                 retval = -1;
2279                 goto out_unlock;
2280         }
2281         memcpy(facts_data, (u8 *)data, data_len);
2282 out_unlock:
2283         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2284         mutex_unlock(&mrioc->init_cmds.mutex);
2285
2286 out:
2287         if (data)
2288                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2289
2290         return retval;
2291 }
2292
2293 /**
2294  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2295  * @mrioc: Adapter instance reference
2296  *
2297  * Check whether the new DMA mask requested through IOCFacts by
2298  * firmware needs to be set, if so set it .
2299  *
2300  * Return: 0 on success, non-zero on failure.
2301  */
2302 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2303 {
2304         struct pci_dev *pdev = mrioc->pdev;
2305         int r;
2306         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2307
2308         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2309                 return 0;
2310
2311         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2312             mrioc->dma_mask, facts_dma_mask);
2313
2314         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2315         if (r) {
2316                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2317                     facts_dma_mask, r);
2318                 return r;
2319         }
2320         mrioc->dma_mask = facts_dma_mask;
2321         return r;
2322 }
2323
2324 /**
2325  * mpi3mr_process_factsdata - Process IOC facts data
2326  * @mrioc: Adapter instance reference
2327  * @facts_data: Cached IOC facts data
2328  *
2329  * Convert IOC facts data into cpu endianness and cache it in
2330  * the driver .
2331  *
2332  * Return: Nothing.
2333  */
2334 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2335         struct mpi3_ioc_facts_data *facts_data)
2336 {
2337         u32 ioc_config, req_sz, facts_flags;
2338
2339         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2340             (sizeof(*facts_data) / 4)) {
2341                 ioc_warn(mrioc,
2342                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2343                     sizeof(*facts_data),
2344                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2345         }
2346
2347         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2348         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2349             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2350         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2351                 ioc_err(mrioc,
2352                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2353                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2354         }
2355
2356         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2357
2358         facts_flags = le32_to_cpu(facts_data->flags);
2359         mrioc->facts.op_req_sz = req_sz;
2360         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2361             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2362             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2363
2364         mrioc->facts.ioc_num = facts_data->ioc_number;
2365         mrioc->facts.who_init = facts_data->who_init;
2366         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2367         mrioc->facts.personality = (facts_flags &
2368             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2369         mrioc->facts.dma_mask = (facts_flags &
2370             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2371             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2372         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2373         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2374         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_request);
2375         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2376         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2377         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2378         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2379         mrioc->facts.max_pds = le16_to_cpu(facts_data->max_pds);
2380         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2381         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2382         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_advanced_host_pds);
2383         mrioc->facts.max_raidpds = le16_to_cpu(facts_data->max_raid_pds);
2384         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2385         mrioc->facts.max_pcie_switches =
2386             le16_to_cpu(facts_data->max_pc_ie_switches);
2387         mrioc->facts.max_sasexpanders =
2388             le16_to_cpu(facts_data->max_sas_expanders);
2389         mrioc->facts.max_sasinitiators =
2390             le16_to_cpu(facts_data->max_sas_initiators);
2391         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2392         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2393         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2394         mrioc->facts.max_op_req_q =
2395             le16_to_cpu(facts_data->max_operational_request_queues);
2396         mrioc->facts.max_op_reply_q =
2397             le16_to_cpu(facts_data->max_operational_reply_queues);
2398         mrioc->facts.ioc_capabilities =
2399             le32_to_cpu(facts_data->ioc_capabilities);
2400         mrioc->facts.fw_ver.build_num =
2401             le16_to_cpu(facts_data->fw_version.build_num);
2402         mrioc->facts.fw_ver.cust_id =
2403             le16_to_cpu(facts_data->fw_version.customer_id);
2404         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2405         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2406         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2407         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2408         mrioc->msix_count = min_t(int, mrioc->msix_count,
2409             mrioc->facts.max_msix_vectors);
2410         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2411         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2412         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2413         mrioc->facts.shutdown_timeout =
2414             le16_to_cpu(facts_data->shutdown_timeout);
2415
2416         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2417             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2418             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2419         ioc_info(mrioc,
2420             "maxreqs(%d), mindh(%d) maxPDs(%d) maxvectors(%d) maxperids(%d)\n",
2421             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2422             mrioc->facts.max_pds, mrioc->facts.max_msix_vectors,
2423             mrioc->facts.max_perids);
2424         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2425             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2426             mrioc->facts.sge_mod_shift);
2427         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2428             mrioc->facts.dma_mask, (facts_flags &
2429             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2430
2431         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
2432
2433         if (reset_devices)
2434                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
2435                     MPI3MR_HOST_IOS_KDUMP);
2436 }
2437
2438 /**
2439  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2440  * @mrioc: Adapter instance reference
2441  *
2442  * Allocate and initialize the reply free buffers, sense
2443  * buffers, reply free queue and sense buffer queue.
2444  *
2445  * Return: 0 on success, non-zero on failures.
2446  */
2447 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2448 {
2449         int retval = 0;
2450         u32 sz, i;
2451         dma_addr_t phy_addr;
2452
2453         if (mrioc->init_cmds.reply)
2454                 goto post_reply_sbuf;
2455
2456         mrioc->init_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2457         if (!mrioc->init_cmds.reply)
2458                 goto out_failed;
2459
2460         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2461                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->facts.reply_sz,
2462                     GFP_KERNEL);
2463                 if (!mrioc->dev_rmhs_cmds[i].reply)
2464                         goto out_failed;
2465         }
2466
2467         mrioc->host_tm_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2468         if (!mrioc->host_tm_cmds.reply)
2469                 goto out_failed;
2470
2471         mrioc->dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
2472         if (mrioc->facts.max_devhandle % 8)
2473                 mrioc->dev_handle_bitmap_sz++;
2474         mrioc->removepend_bitmap = kzalloc(mrioc->dev_handle_bitmap_sz,
2475             GFP_KERNEL);
2476         if (!mrioc->removepend_bitmap)
2477                 goto out_failed;
2478
2479         mrioc->devrem_bitmap_sz = MPI3MR_NUM_DEVRMCMD / 8;
2480         if (MPI3MR_NUM_DEVRMCMD % 8)
2481                 mrioc->devrem_bitmap_sz++;
2482         mrioc->devrem_bitmap = kzalloc(mrioc->devrem_bitmap_sz,
2483             GFP_KERNEL);
2484         if (!mrioc->devrem_bitmap)
2485                 goto out_failed;
2486
2487         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2488         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2489         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2490         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2491
2492         /* reply buffer pool, 16 byte align */
2493         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2494         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2495             &mrioc->pdev->dev, sz, 16, 0);
2496         if (!mrioc->reply_buf_pool) {
2497                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2498                 goto out_failed;
2499         }
2500
2501         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2502             &mrioc->reply_buf_dma);
2503         if (!mrioc->reply_buf)
2504                 goto out_failed;
2505
2506         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2507
2508         /* reply free queue, 8 byte align */
2509         sz = mrioc->reply_free_qsz * 8;
2510         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2511             &mrioc->pdev->dev, sz, 8, 0);
2512         if (!mrioc->reply_free_q_pool) {
2513                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2514                 goto out_failed;
2515         }
2516         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
2517             GFP_KERNEL, &mrioc->reply_free_q_dma);
2518         if (!mrioc->reply_free_q)
2519                 goto out_failed;
2520
2521         /* sense buffer pool,  4 byte align */
2522         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2523         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
2524             &mrioc->pdev->dev, sz, 4, 0);
2525         if (!mrioc->sense_buf_pool) {
2526                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
2527                 goto out_failed;
2528         }
2529         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
2530             &mrioc->sense_buf_dma);
2531         if (!mrioc->sense_buf)
2532                 goto out_failed;
2533
2534         /* sense buffer queue, 8 byte align */
2535         sz = mrioc->sense_buf_q_sz * 8;
2536         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
2537             &mrioc->pdev->dev, sz, 8, 0);
2538         if (!mrioc->sense_buf_q_pool) {
2539                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
2540                 goto out_failed;
2541         }
2542         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
2543             GFP_KERNEL, &mrioc->sense_buf_q_dma);
2544         if (!mrioc->sense_buf_q)
2545                 goto out_failed;
2546
2547 post_reply_sbuf:
2548         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2549         ioc_info(mrioc,
2550             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2551             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->facts.reply_sz,
2552             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
2553         sz = mrioc->reply_free_qsz * 8;
2554         ioc_info(mrioc,
2555             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2556             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
2557             (unsigned long long)mrioc->reply_free_q_dma);
2558         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2559         ioc_info(mrioc,
2560             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2561             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSEBUF_SZ,
2562             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
2563         sz = mrioc->sense_buf_q_sz * 8;
2564         ioc_info(mrioc,
2565             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2566             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
2567             (unsigned long long)mrioc->sense_buf_q_dma);
2568
2569         /* initialize Reply buffer Queue */
2570         for (i = 0, phy_addr = mrioc->reply_buf_dma;
2571             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->facts.reply_sz)
2572                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
2573         mrioc->reply_free_q[i] = cpu_to_le64(0);
2574
2575         /* initialize Sense Buffer Queue */
2576         for (i = 0, phy_addr = mrioc->sense_buf_dma;
2577             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSEBUF_SZ)
2578                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
2579         mrioc->sense_buf_q[i] = cpu_to_le64(0);
2580         return retval;
2581
2582 out_failed:
2583         retval = -1;
2584         return retval;
2585 }
2586
2587 /**
2588  * mpi3mr_issue_iocinit - Send IOC Init
2589  * @mrioc: Adapter instance reference
2590  *
2591  * Issue IOC Init MPI request through admin queue and wait for
2592  * the completion of it or time out.
2593  *
2594  * Return: 0 on success, non-zero on failures.
2595  */
2596 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
2597 {
2598         struct mpi3_ioc_init_request iocinit_req;
2599         struct mpi3_driver_info_layout *drv_info;
2600         dma_addr_t data_dma;
2601         u32 data_len = sizeof(*drv_info);
2602         int retval = 0;
2603         ktime_t current_time;
2604
2605         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2606             GFP_KERNEL);
2607         if (!drv_info) {
2608                 retval = -1;
2609                 goto out;
2610         }
2611         drv_info->information_length = cpu_to_le32(data_len);
2612         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
2613         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
2614         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
2615         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
2616         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
2617         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
2618             sizeof(drv_info->driver_release_date));
2619         drv_info->driver_capabilities = 0;
2620         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
2621             sizeof(mrioc->driver_info));
2622
2623         memset(&iocinit_req, 0, sizeof(iocinit_req));
2624         mutex_lock(&mrioc->init_cmds.mutex);
2625         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2626                 retval = -1;
2627                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
2628                 mutex_unlock(&mrioc->init_cmds.mutex);
2629                 goto out;
2630         }
2631         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2632         mrioc->init_cmds.is_waiting = 1;
2633         mrioc->init_cmds.callback = NULL;
2634         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2635         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
2636         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
2637         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
2638         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
2639         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
2640         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
2641         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
2642         iocinit_req.reply_free_queue_address =
2643             cpu_to_le64(mrioc->reply_free_q_dma);
2644         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSEBUF_SZ);
2645         iocinit_req.sense_buffer_free_queue_depth =
2646             cpu_to_le16(mrioc->sense_buf_q_sz);
2647         iocinit_req.sense_buffer_free_queue_address =
2648             cpu_to_le64(mrioc->sense_buf_q_dma);
2649         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
2650
2651         current_time = ktime_get_real();
2652         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
2653
2654         init_completion(&mrioc->init_cmds.done);
2655         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
2656             sizeof(iocinit_req), 1);
2657         if (retval) {
2658                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
2659                 goto out_unlock;
2660         }
2661         wait_for_completion_timeout(&mrioc->init_cmds.done,
2662             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2663         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2664                 mpi3mr_set_diagsave(mrioc);
2665                 mpi3mr_issue_reset(mrioc,
2666                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2667                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
2668                 mrioc->unrecoverable = 1;
2669                 ioc_err(mrioc, "Issue IOCInit: command timed out\n");
2670                 retval = -1;
2671                 goto out_unlock;
2672         }
2673         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2674             != MPI3_IOCSTATUS_SUCCESS) {
2675                 ioc_err(mrioc,
2676                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2677                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2678                     mrioc->init_cmds.ioc_loginfo);
2679                 retval = -1;
2680                 goto out_unlock;
2681         }
2682
2683 out_unlock:
2684         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2685         mutex_unlock(&mrioc->init_cmds.mutex);
2686
2687 out:
2688         if (drv_info)
2689                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
2690                     data_dma);
2691
2692         return retval;
2693 }
2694
2695 /**
2696  * mpi3mr_unmask_events - Unmask events in event mask bitmap
2697  * @mrioc: Adapter instance reference
2698  * @event: MPI event ID
2699  *
2700  * Un mask the specific event by resetting the event_mask
2701  * bitmap.
2702  *
2703  * Return: 0 on success, non-zero on failures.
2704  */
2705 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
2706 {
2707         u32 desired_event;
2708         u8 word;
2709
2710         if (event >= 128)
2711                 return;
2712
2713         desired_event = (1 << (event % 32));
2714         word = event / 32;
2715
2716         mrioc->event_masks[word] &= ~desired_event;
2717 }
2718
2719 /**
2720  * mpi3mr_issue_event_notification - Send event notification
2721  * @mrioc: Adapter instance reference
2722  *
2723  * Issue event notification MPI request through admin queue and
2724  * wait for the completion of it or time out.
2725  *
2726  * Return: 0 on success, non-zero on failures.
2727  */
2728 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
2729 {
2730         struct mpi3_event_notification_request evtnotify_req;
2731         int retval = 0;
2732         u8 i;
2733
2734         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
2735         mutex_lock(&mrioc->init_cmds.mutex);
2736         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2737                 retval = -1;
2738                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
2739                 mutex_unlock(&mrioc->init_cmds.mutex);
2740                 goto out;
2741         }
2742         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2743         mrioc->init_cmds.is_waiting = 1;
2744         mrioc->init_cmds.callback = NULL;
2745         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2746         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
2747         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2748                 evtnotify_req.event_masks[i] =
2749                     cpu_to_le32(mrioc->event_masks[i]);
2750         init_completion(&mrioc->init_cmds.done);
2751         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
2752             sizeof(evtnotify_req), 1);
2753         if (retval) {
2754                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
2755                 goto out_unlock;
2756         }
2757         wait_for_completion_timeout(&mrioc->init_cmds.done,
2758             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2759         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2760                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2761                 mpi3mr_set_diagsave(mrioc);
2762                 mpi3mr_issue_reset(mrioc,
2763                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2764                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
2765                 mrioc->unrecoverable = 1;
2766                 retval = -1;
2767                 goto out_unlock;
2768         }
2769         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2770             != MPI3_IOCSTATUS_SUCCESS) {
2771                 ioc_err(mrioc,
2772                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2773                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2774                     mrioc->init_cmds.ioc_loginfo);
2775                 retval = -1;
2776                 goto out_unlock;
2777         }
2778
2779 out_unlock:
2780         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2781         mutex_unlock(&mrioc->init_cmds.mutex);
2782 out:
2783         return retval;
2784 }
2785
2786 /**
2787  * mpi3mr_send_event_ack - Send event acknowledgment
2788  * @mrioc: Adapter instance reference
2789  * @event: MPI3 event ID
2790  * @event_ctx: Event context
2791  *
2792  * Send event acknowledgment through admin queue and wait for
2793  * it to complete.
2794  *
2795  * Return: 0 on success, non-zero on failures.
2796  */
2797 int mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2798         u32 event_ctx)
2799 {
2800         struct mpi3_event_ack_request evtack_req;
2801         int retval = 0;
2802
2803         memset(&evtack_req, 0, sizeof(evtack_req));
2804         mutex_lock(&mrioc->init_cmds.mutex);
2805         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2806                 retval = -1;
2807                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
2808                 mutex_unlock(&mrioc->init_cmds.mutex);
2809                 goto out;
2810         }
2811         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2812         mrioc->init_cmds.is_waiting = 1;
2813         mrioc->init_cmds.callback = NULL;
2814         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2815         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2816         evtack_req.event = event;
2817         evtack_req.event_context = cpu_to_le32(event_ctx);
2818
2819         init_completion(&mrioc->init_cmds.done);
2820         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2821             sizeof(evtack_req), 1);
2822         if (retval) {
2823                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
2824                 goto out_unlock;
2825         }
2826         wait_for_completion_timeout(&mrioc->init_cmds.done,
2827             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2828         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2829                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2830                 mpi3mr_soft_reset_handler(mrioc,
2831                     MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
2832                 retval = -1;
2833                 goto out_unlock;
2834         }
2835         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2836             != MPI3_IOCSTATUS_SUCCESS) {
2837                 ioc_err(mrioc,
2838                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2839                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2840                     mrioc->init_cmds.ioc_loginfo);
2841                 retval = -1;
2842                 goto out_unlock;
2843         }
2844
2845 out_unlock:
2846         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2847         mutex_unlock(&mrioc->init_cmds.mutex);
2848 out:
2849         return retval;
2850 }
2851
2852 /**
2853  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
2854  * @mrioc: Adapter instance reference
2855  *
2856  * Allocate chain buffers and set a bitmap to indicate free
2857  * chain buffers. Chain buffers are used to pass the SGE
2858  * information along with MPI3 SCSI IO requests for host I/O.
2859  *
2860  * Return: 0 on success, non-zero on failure
2861  */
2862 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
2863 {
2864         int retval = 0;
2865         u32 sz, i;
2866         u16 num_chains;
2867
2868         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
2869
2870         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
2871             | SHOST_DIX_TYPE1_PROTECTION
2872             | SHOST_DIX_TYPE2_PROTECTION
2873             | SHOST_DIX_TYPE3_PROTECTION))
2874                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
2875
2876         mrioc->chain_buf_count = num_chains;
2877         sz = sizeof(struct chain_element) * num_chains;
2878         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
2879         if (!mrioc->chain_sgl_list)
2880                 goto out_failed;
2881
2882         sz = MPI3MR_PAGE_SIZE_4K;
2883         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
2884             &mrioc->pdev->dev, sz, 16, 0);
2885         if (!mrioc->chain_buf_pool) {
2886                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
2887                 goto out_failed;
2888         }
2889
2890         for (i = 0; i < num_chains; i++) {
2891                 mrioc->chain_sgl_list[i].addr =
2892                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
2893                     &mrioc->chain_sgl_list[i].dma_addr);
2894
2895                 if (!mrioc->chain_sgl_list[i].addr)
2896                         goto out_failed;
2897         }
2898         mrioc->chain_bitmap_sz = num_chains / 8;
2899         if (num_chains % 8)
2900                 mrioc->chain_bitmap_sz++;
2901         mrioc->chain_bitmap = kzalloc(mrioc->chain_bitmap_sz, GFP_KERNEL);
2902         if (!mrioc->chain_bitmap)
2903                 goto out_failed;
2904         return retval;
2905 out_failed:
2906         retval = -1;
2907         return retval;
2908 }
2909
2910 /**
2911  * mpi3mr_port_enable_complete - Mark port enable complete
2912  * @mrioc: Adapter instance reference
2913  * @drv_cmd: Internal command tracker
2914  *
2915  * Call back for asynchronous port enable request sets the
2916  * driver command to indicate port enable request is complete.
2917  *
2918  * Return: Nothing
2919  */
2920 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
2921         struct mpi3mr_drv_cmd *drv_cmd)
2922 {
2923         drv_cmd->state = MPI3MR_CMD_NOTUSED;
2924         drv_cmd->callback = NULL;
2925         mrioc->scan_failed = drv_cmd->ioc_status;
2926         mrioc->scan_started = 0;
2927 }
2928
2929 /**
2930  * mpi3mr_issue_port_enable - Issue Port Enable
2931  * @mrioc: Adapter instance reference
2932  * @async: Flag to wait for completion or not
2933  *
2934  * Issue Port Enable MPI request through admin queue and if the
2935  * async flag is not set wait for the completion of the port
2936  * enable or time out.
2937  *
2938  * Return: 0 on success, non-zero on failures.
2939  */
2940 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
2941 {
2942         struct mpi3_port_enable_request pe_req;
2943         int retval = 0;
2944         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
2945
2946         memset(&pe_req, 0, sizeof(pe_req));
2947         mutex_lock(&mrioc->init_cmds.mutex);
2948         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2949                 retval = -1;
2950                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
2951                 mutex_unlock(&mrioc->init_cmds.mutex);
2952                 goto out;
2953         }
2954         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2955         if (async) {
2956                 mrioc->init_cmds.is_waiting = 0;
2957                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
2958         } else {
2959                 mrioc->init_cmds.is_waiting = 1;
2960                 mrioc->init_cmds.callback = NULL;
2961                 init_completion(&mrioc->init_cmds.done);
2962         }
2963         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2964         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
2965
2966         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
2967         if (retval) {
2968                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
2969                 goto out_unlock;
2970         }
2971         if (!async) {
2972                 wait_for_completion_timeout(&mrioc->init_cmds.done,
2973                     (pe_timeout * HZ));
2974                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2975                         ioc_err(mrioc, "Issue PortEnable: command timed out\n");
2976                         retval = -1;
2977                         mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
2978                         mpi3mr_set_diagsave(mrioc);
2979                         mpi3mr_issue_reset(mrioc,
2980                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2981                             MPI3MR_RESET_FROM_PE_TIMEOUT);
2982                         mrioc->unrecoverable = 1;
2983                         goto out_unlock;
2984                 }
2985                 mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
2986         }
2987 out_unlock:
2988         mutex_unlock(&mrioc->init_cmds.mutex);
2989 out:
2990         return retval;
2991 }
2992
2993 /* Protocol type to name mapper structure*/
2994 static const struct {
2995         u8 protocol;
2996         char *name;
2997 } mpi3mr_protocols[] = {
2998         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
2999         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3000         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3001 };
3002
3003 /* Capability to name mapper structure*/
3004 static const struct {
3005         u32 capability;
3006         char *name;
3007 } mpi3mr_capabilities[] = {
3008         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3009 };
3010
3011 /**
3012  * mpi3mr_print_ioc_info - Display controller information
3013  * @mrioc: Adapter instance reference
3014  *
3015  * Display controller personalit, capability, supported
3016  * protocols etc.
3017  *
3018  * Return: Nothing
3019  */
3020 static void
3021 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3022 {
3023         int i = 0, bytes_wrote = 0;
3024         char personality[16];
3025         char protocol[50] = {0};
3026         char capabilities[100] = {0};
3027         bool is_string_nonempty = false;
3028         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3029
3030         switch (mrioc->facts.personality) {
3031         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3032                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3033                 break;
3034         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3035                 strncpy(personality, "RAID", sizeof(personality));
3036                 break;
3037         default:
3038                 strncpy(personality, "Unknown", sizeof(personality));
3039                 break;
3040         }
3041
3042         ioc_info(mrioc, "Running in %s Personality", personality);
3043
3044         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3045             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3046             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3047
3048         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3049                 if (mrioc->facts.protocol_flags &
3050                     mpi3mr_protocols[i].protocol) {
3051                         if (is_string_nonempty &&
3052                             (bytes_wrote < sizeof(protocol)))
3053                                 bytes_wrote += snprintf(protocol + bytes_wrote,
3054                                     (sizeof(protocol) - bytes_wrote), ",");
3055
3056                         if (bytes_wrote < sizeof(protocol))
3057                                 bytes_wrote += snprintf(protocol + bytes_wrote,
3058                                     (sizeof(protocol) - bytes_wrote), "%s",
3059                                     mpi3mr_protocols[i].name);
3060                         is_string_nonempty = true;
3061                 }
3062         }
3063
3064         bytes_wrote = 0;
3065         is_string_nonempty = false;
3066         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3067                 if (mrioc->facts.protocol_flags &
3068                     mpi3mr_capabilities[i].capability) {
3069                         if (is_string_nonempty &&
3070                             (bytes_wrote < sizeof(capabilities)))
3071                                 bytes_wrote += snprintf(capabilities + bytes_wrote,
3072                                     (sizeof(capabilities) - bytes_wrote), ",");
3073
3074                         if (bytes_wrote < sizeof(capabilities))
3075                                 bytes_wrote += snprintf(capabilities + bytes_wrote,
3076                                     (sizeof(capabilities) - bytes_wrote), "%s",
3077                                     mpi3mr_capabilities[i].name);
3078                         is_string_nonempty = true;
3079                 }
3080         }
3081
3082         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3083             protocol, capabilities);
3084 }
3085
3086 /**
3087  * mpi3mr_cleanup_resources - Free PCI resources
3088  * @mrioc: Adapter instance reference
3089  *
3090  * Unmap PCI device memory and disable PCI device.
3091  *
3092  * Return: 0 on success and non-zero on failure.
3093  */
3094 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3095 {
3096         struct pci_dev *pdev = mrioc->pdev;
3097
3098         mpi3mr_cleanup_isr(mrioc);
3099
3100         if (mrioc->sysif_regs) {
3101                 iounmap((void __iomem *)mrioc->sysif_regs);
3102                 mrioc->sysif_regs = NULL;
3103         }
3104
3105         if (pci_is_enabled(pdev)) {
3106                 if (mrioc->bars)
3107                         pci_release_selected_regions(pdev, mrioc->bars);
3108                 pci_disable_device(pdev);
3109         }
3110 }
3111
3112 /**
3113  * mpi3mr_setup_resources - Enable PCI resources
3114  * @mrioc: Adapter instance reference
3115  *
3116  * Enable PCI device memory, MSI-x registers and set DMA mask.
3117  *
3118  * Return: 0 on success and non-zero on failure.
3119  */
3120 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3121 {
3122         struct pci_dev *pdev = mrioc->pdev;
3123         u32 memap_sz = 0;
3124         int i, retval = 0, capb = 0;
3125         u16 message_control;
3126         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3127             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3128             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3129
3130         if (pci_enable_device_mem(pdev)) {
3131                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3132                 retval = -ENODEV;
3133                 goto out_failed;
3134         }
3135
3136         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3137         if (!capb) {
3138                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3139                 retval = -ENODEV;
3140                 goto out_failed;
3141         }
3142         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3143
3144         if (pci_request_selected_regions(pdev, mrioc->bars,
3145             mrioc->driver_name)) {
3146                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3147                 retval = -ENODEV;
3148                 goto out_failed;
3149         }
3150
3151         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3152                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3153                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3154                         memap_sz = pci_resource_len(pdev, i);
3155                         mrioc->sysif_regs =
3156                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3157                         break;
3158                 }
3159         }
3160
3161         pci_set_master(pdev);
3162
3163         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3164         if (retval) {
3165                 if (dma_mask != DMA_BIT_MASK(32)) {
3166                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3167                         dma_mask = DMA_BIT_MASK(32);
3168                         retval = dma_set_mask_and_coherent(&pdev->dev,
3169                             dma_mask);
3170                 }
3171                 if (retval) {
3172                         mrioc->dma_mask = 0;
3173                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3174                         goto out_failed;
3175                 }
3176         }
3177         mrioc->dma_mask = dma_mask;
3178
3179         if (!mrioc->sysif_regs) {
3180                 ioc_err(mrioc,
3181                     "Unable to map adapter memory or resource not found\n");
3182                 retval = -EINVAL;
3183                 goto out_failed;
3184         }
3185
3186         pci_read_config_word(pdev, capb + 2, &message_control);
3187         mrioc->msix_count = (message_control & 0x3FF) + 1;
3188
3189         pci_save_state(pdev);
3190
3191         pci_set_drvdata(pdev, mrioc->shost);
3192
3193         mpi3mr_ioc_disable_intr(mrioc);
3194
3195         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3196             (unsigned long long)mrioc->sysif_regs_phys,
3197             mrioc->sysif_regs, memap_sz);
3198         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3199             mrioc->msix_count);
3200         return retval;
3201
3202 out_failed:
3203         mpi3mr_cleanup_resources(mrioc);
3204         return retval;
3205 }
3206
3207 /**
3208  * mpi3mr_init_ioc - Initialize the controller
3209  * @mrioc: Adapter instance reference
3210  * @init_type: Flag to indicate is the init_type
3211  *
3212  * This the controller initialization routine, executed either
3213  * after soft reset or from pci probe callback.
3214  * Setup the required resources, memory map the controller
3215  * registers, create admin and operational reply queue pairs,
3216  * allocate required memory for reply pool, sense buffer pool,
3217  * issue IOC init request to the firmware, unmask the events and
3218  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3219  * volumes.
3220  *
3221  * Return: 0 on success and non-zero on failure.
3222  */
3223 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc, u8 init_type)
3224 {
3225         int retval = 0;
3226         enum mpi3mr_iocstate ioc_state;
3227         u64 base_info;
3228         u32 timeout;
3229         u32 ioc_status, ioc_config, i;
3230         struct mpi3_ioc_facts_data facts_data;
3231
3232         mrioc->irqpoll_sleep = MPI3MR_IRQ_POLL_SLEEP;
3233         mrioc->change_count = 0;
3234         if (init_type == MPI3MR_IT_INIT) {
3235                 mrioc->cpu_count = num_online_cpus();
3236                 retval = mpi3mr_setup_resources(mrioc);
3237                 if (retval) {
3238                         ioc_err(mrioc, "Failed to setup resources:error %d\n",
3239                             retval);
3240                         goto out_nocleanup;
3241                 }
3242         }
3243
3244         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3245         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3246
3247         ioc_info(mrioc, "SOD status %x configuration %x\n",
3248             ioc_status, ioc_config);
3249
3250         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
3251         ioc_info(mrioc, "SOD base_info %llx\n", base_info);
3252
3253         /*The timeout value is in 2sec unit, changing it to seconds*/
3254         mrioc->ready_timeout =
3255             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
3256             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
3257
3258         ioc_info(mrioc, "IOC ready timeout %d\n", mrioc->ready_timeout);
3259
3260         ioc_state = mpi3mr_get_iocstate(mrioc);
3261         ioc_info(mrioc, "IOC in %s state during detection\n",
3262             mpi3mr_iocstate_name(ioc_state));
3263
3264         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
3265             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
3266                 timeout = mrioc->ready_timeout * 10;
3267                 do {
3268                         msleep(100);
3269                 } while (--timeout);
3270
3271                 ioc_state = mpi3mr_get_iocstate(mrioc);
3272                 ioc_info(mrioc,
3273                     "IOC in %s state after waiting for reset time\n",
3274                     mpi3mr_iocstate_name(ioc_state));
3275         }
3276
3277         if (ioc_state == MRIOC_STATE_READY) {
3278                 retval = mpi3mr_issue_and_process_mur(mrioc,
3279                     MPI3MR_RESET_FROM_BRINGUP);
3280                 if (retval) {
3281                         ioc_err(mrioc, "Failed to MU reset IOC error %d\n",
3282                             retval);
3283                 }
3284                 ioc_state = mpi3mr_get_iocstate(mrioc);
3285         }
3286         if (ioc_state != MRIOC_STATE_RESET) {
3287                 mpi3mr_print_fault_info(mrioc);
3288                 retval = mpi3mr_issue_reset(mrioc,
3289                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3290                     MPI3MR_RESET_FROM_BRINGUP);
3291                 if (retval) {
3292                         ioc_err(mrioc,
3293                             "%s :Failed to soft reset IOC error %d\n",
3294                             __func__, retval);
3295                         goto out_failed;
3296                 }
3297         }
3298         ioc_state = mpi3mr_get_iocstate(mrioc);
3299         if (ioc_state != MRIOC_STATE_RESET) {
3300                 retval = -1;
3301                 ioc_err(mrioc, "Cannot bring IOC to reset state\n");
3302                 goto out_failed;
3303         }
3304
3305         retval = mpi3mr_setup_admin_qpair(mrioc);
3306         if (retval) {
3307                 ioc_err(mrioc, "Failed to setup admin Qs: error %d\n",
3308                     retval);
3309                 goto out_failed;
3310         }
3311
3312         retval = mpi3mr_bring_ioc_ready(mrioc);
3313         if (retval) {
3314                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3315                     retval);
3316                 goto out_failed;
3317         }
3318
3319         if (init_type != MPI3MR_IT_RESET) {
3320                 retval = mpi3mr_setup_isr(mrioc, 1);
3321                 if (retval) {
3322                         ioc_err(mrioc, "Failed to setup ISR error %d\n",
3323                             retval);
3324                         goto out_failed;
3325                 }
3326         } else
3327                 mpi3mr_ioc_enable_intr(mrioc);
3328
3329         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3330         if (retval) {
3331                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3332                     retval);
3333                 goto out_failed;
3334         }
3335
3336         mpi3mr_process_factsdata(mrioc, &facts_data);
3337         if (init_type == MPI3MR_IT_INIT) {
3338                 retval = mpi3mr_check_reset_dma_mask(mrioc);
3339                 if (retval) {
3340                         ioc_err(mrioc, "Resetting dma mask failed %d\n",
3341                             retval);
3342                         goto out_failed;
3343                 }
3344         }
3345
3346         mpi3mr_print_ioc_info(mrioc);
3347
3348         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3349         if (retval) {
3350                 ioc_err(mrioc,
3351                     "%s :Failed to allocated reply sense buffers %d\n",
3352                     __func__, retval);
3353                 goto out_failed;
3354         }
3355
3356         if (init_type == MPI3MR_IT_INIT) {
3357                 retval = mpi3mr_alloc_chain_bufs(mrioc);
3358                 if (retval) {
3359                         ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3360                             retval);
3361                         goto out_failed;
3362                 }
3363         }
3364
3365         retval = mpi3mr_issue_iocinit(mrioc);
3366         if (retval) {
3367                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3368                     retval);
3369                 goto out_failed;
3370         }
3371         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3372         writel(mrioc->reply_free_queue_host_index,
3373             &mrioc->sysif_regs->reply_free_host_index);
3374
3375         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3376         writel(mrioc->sbq_host_index,
3377             &mrioc->sysif_regs->sense_buffer_free_host_index);
3378
3379         if (init_type != MPI3MR_IT_RESET) {
3380                 retval = mpi3mr_setup_isr(mrioc, 0);
3381                 if (retval) {
3382                         ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3383                             retval);
3384                         goto out_failed;
3385                 }
3386         }
3387
3388         retval = mpi3mr_create_op_queues(mrioc);
3389         if (retval) {
3390                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3391                     retval);
3392                 goto out_failed;
3393         }
3394
3395         if ((init_type != MPI3MR_IT_INIT) &&
3396             (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q)) {
3397                 retval = -1;
3398                 ioc_err(mrioc,
3399                     "Cannot create minimum number of OpQueues expected:%d created:%d\n",
3400                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
3401                 goto out_failed;
3402         }
3403
3404         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3405                 mrioc->event_masks[i] = -1;
3406
3407         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3408         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3409         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3410         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3411         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3412         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3413         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3414         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3415         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3416         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3417         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3418         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3419
3420         retval = mpi3mr_issue_event_notification(mrioc);
3421         if (retval) {
3422                 ioc_err(mrioc, "Failed to issue event notification %d\n",
3423                     retval);
3424                 goto out_failed;
3425         }
3426
3427         if (init_type != MPI3MR_IT_INIT) {
3428                 ioc_info(mrioc, "Issuing Port Enable\n");
3429                 retval = mpi3mr_issue_port_enable(mrioc, 0);
3430                 if (retval) {
3431                         ioc_err(mrioc, "Failed to issue port enable %d\n",
3432                             retval);
3433                         goto out_failed;
3434                 }
3435         }
3436         return retval;
3437
3438 out_failed:
3439         if (init_type == MPI3MR_IT_INIT)
3440                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_COMPLETE_CLEANUP);
3441         else
3442                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_REINIT_FAILURE);
3443 out_nocleanup:
3444         return retval;
3445 }
3446
3447 /**
3448  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
3449  *                                      segments
3450  * @mrioc: Adapter instance reference
3451  * @qidx: Operational reply queue index
3452  *
3453  * Return: Nothing.
3454  */
3455 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3456 {
3457         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
3458         struct segments *segments;
3459         int i, size;
3460
3461         if (!op_reply_q->q_segments)
3462                 return;
3463
3464         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
3465         segments = op_reply_q->q_segments;
3466         for (i = 0; i < op_reply_q->num_segments; i++)
3467                 memset(segments[i].segment, 0, size);
3468 }
3469
3470 /**
3471  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
3472  *                                      segments
3473  * @mrioc: Adapter instance reference
3474  * @qidx: Operational request queue index
3475  *
3476  * Return: Nothing.
3477  */
3478 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3479 {
3480         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
3481         struct segments *segments;
3482         int i, size;
3483
3484         if (!op_req_q->q_segments)
3485                 return;
3486
3487         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
3488         segments = op_req_q->q_segments;
3489         for (i = 0; i < op_req_q->num_segments; i++)
3490                 memset(segments[i].segment, 0, size);
3491 }
3492
3493 /**
3494  * mpi3mr_memset_buffers - memset memory for a controller
3495  * @mrioc: Adapter instance reference
3496  *
3497  * clear all the memory allocated for a controller, typically
3498  * called post reset to reuse the memory allocated during the
3499  * controller init.
3500  *
3501  * Return: Nothing.
3502  */
3503 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
3504 {
3505         u16 i;
3506
3507         memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
3508         memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
3509
3510         memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
3511         memset(mrioc->host_tm_cmds.reply, 0,
3512             sizeof(*mrioc->host_tm_cmds.reply));
3513         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
3514                 memset(mrioc->dev_rmhs_cmds[i].reply, 0,
3515                     sizeof(*mrioc->dev_rmhs_cmds[i].reply));
3516         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3517         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3518
3519         for (i = 0; i < mrioc->num_queues; i++) {
3520                 mrioc->op_reply_qinfo[i].qid = 0;
3521                 mrioc->op_reply_qinfo[i].ci = 0;
3522                 mrioc->op_reply_qinfo[i].num_replies = 0;
3523                 mrioc->op_reply_qinfo[i].ephase = 0;
3524                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
3525                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
3526                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
3527
3528                 mrioc->req_qinfo[i].ci = 0;
3529                 mrioc->req_qinfo[i].pi = 0;
3530                 mrioc->req_qinfo[i].num_requests = 0;
3531                 mrioc->req_qinfo[i].qid = 0;
3532                 mrioc->req_qinfo[i].reply_qid = 0;
3533                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
3534                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
3535         }
3536 }
3537
3538 /**
3539  * mpi3mr_free_mem - Free memory allocated for a controller
3540  * @mrioc: Adapter instance reference
3541  *
3542  * Free all the memory allocated for a controller.
3543  *
3544  * Return: Nothing.
3545  */
3546 static void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
3547 {
3548         u16 i;
3549         struct mpi3mr_intr_info *intr_info;
3550
3551         if (mrioc->sense_buf_pool) {
3552                 if (mrioc->sense_buf)
3553                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
3554                             mrioc->sense_buf_dma);
3555                 dma_pool_destroy(mrioc->sense_buf_pool);
3556                 mrioc->sense_buf = NULL;
3557                 mrioc->sense_buf_pool = NULL;
3558         }
3559         if (mrioc->sense_buf_q_pool) {
3560                 if (mrioc->sense_buf_q)
3561                         dma_pool_free(mrioc->sense_buf_q_pool,
3562                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
3563                 dma_pool_destroy(mrioc->sense_buf_q_pool);
3564                 mrioc->sense_buf_q = NULL;
3565                 mrioc->sense_buf_q_pool = NULL;
3566         }
3567
3568         if (mrioc->reply_buf_pool) {
3569                 if (mrioc->reply_buf)
3570                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
3571                             mrioc->reply_buf_dma);
3572                 dma_pool_destroy(mrioc->reply_buf_pool);
3573                 mrioc->reply_buf = NULL;
3574                 mrioc->reply_buf_pool = NULL;
3575         }
3576         if (mrioc->reply_free_q_pool) {
3577                 if (mrioc->reply_free_q)
3578                         dma_pool_free(mrioc->reply_free_q_pool,
3579                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
3580                 dma_pool_destroy(mrioc->reply_free_q_pool);
3581                 mrioc->reply_free_q = NULL;
3582                 mrioc->reply_free_q_pool = NULL;
3583         }
3584
3585         for (i = 0; i < mrioc->num_op_req_q; i++)
3586                 mpi3mr_free_op_req_q_segments(mrioc, i);
3587
3588         for (i = 0; i < mrioc->num_op_reply_q; i++)
3589                 mpi3mr_free_op_reply_q_segments(mrioc, i);
3590
3591         for (i = 0; i < mrioc->intr_info_count; i++) {
3592                 intr_info = mrioc->intr_info + i;
3593                 intr_info->op_reply_q = NULL;
3594         }
3595
3596         kfree(mrioc->req_qinfo);
3597         mrioc->req_qinfo = NULL;
3598         mrioc->num_op_req_q = 0;
3599
3600         kfree(mrioc->op_reply_qinfo);
3601         mrioc->op_reply_qinfo = NULL;
3602         mrioc->num_op_reply_q = 0;
3603
3604         kfree(mrioc->init_cmds.reply);
3605         mrioc->init_cmds.reply = NULL;
3606
3607         kfree(mrioc->host_tm_cmds.reply);
3608         mrioc->host_tm_cmds.reply = NULL;
3609
3610         kfree(mrioc->removepend_bitmap);
3611         mrioc->removepend_bitmap = NULL;
3612
3613         kfree(mrioc->devrem_bitmap);
3614         mrioc->devrem_bitmap = NULL;
3615
3616         kfree(mrioc->chain_bitmap);
3617         mrioc->chain_bitmap = NULL;
3618
3619         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3620                 kfree(mrioc->dev_rmhs_cmds[i].reply);
3621                 mrioc->dev_rmhs_cmds[i].reply = NULL;
3622         }
3623
3624         if (mrioc->chain_buf_pool) {
3625                 for (i = 0; i < mrioc->chain_buf_count; i++) {
3626                         if (mrioc->chain_sgl_list[i].addr) {
3627                                 dma_pool_free(mrioc->chain_buf_pool,
3628                                     mrioc->chain_sgl_list[i].addr,
3629                                     mrioc->chain_sgl_list[i].dma_addr);
3630                                 mrioc->chain_sgl_list[i].addr = NULL;
3631                         }
3632                 }
3633                 dma_pool_destroy(mrioc->chain_buf_pool);
3634                 mrioc->chain_buf_pool = NULL;
3635         }
3636
3637         kfree(mrioc->chain_sgl_list);
3638         mrioc->chain_sgl_list = NULL;
3639
3640         if (mrioc->admin_reply_base) {
3641                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
3642                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
3643                 mrioc->admin_reply_base = NULL;
3644         }
3645         if (mrioc->admin_req_base) {
3646                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
3647                     mrioc->admin_req_base, mrioc->admin_req_dma);
3648                 mrioc->admin_req_base = NULL;
3649         }
3650 }
3651
3652 /**
3653  * mpi3mr_issue_ioc_shutdown - shutdown controller
3654  * @mrioc: Adapter instance reference
3655  *
3656  * Send shutodwn notification to the controller and wait for the
3657  * shutdown_timeout for it to be completed.
3658  *
3659  * Return: Nothing.
3660  */
3661 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
3662 {
3663         u32 ioc_config, ioc_status;
3664         u8 retval = 1;
3665         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
3666
3667         ioc_info(mrioc, "Issuing shutdown Notification\n");
3668         if (mrioc->unrecoverable) {
3669                 ioc_warn(mrioc,
3670                     "IOC is unrecoverable shutdown is not issued\n");
3671                 return;
3672         }
3673         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3674         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3675             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
3676                 ioc_info(mrioc, "shutdown already in progress\n");
3677                 return;
3678         }
3679
3680         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3681         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
3682         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN;
3683
3684         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
3685
3686         if (mrioc->facts.shutdown_timeout)
3687                 timeout = mrioc->facts.shutdown_timeout * 10;
3688
3689         do {
3690                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3691                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3692                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
3693                         retval = 0;
3694                         break;
3695                 }
3696                 msleep(100);
3697         } while (--timeout);
3698
3699         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3700         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3701
3702         if (retval) {
3703                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3704                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
3705                         ioc_warn(mrioc,
3706                             "shutdown still in progress after timeout\n");
3707         }
3708
3709         ioc_info(mrioc,
3710             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
3711             (!retval) ? "successful" : "failed", ioc_status,
3712             ioc_config);
3713 }
3714
3715 /**
3716  * mpi3mr_cleanup_ioc - Cleanup controller
3717  * @mrioc: Adapter instance reference
3718  * @reason: Cleanup reason
3719  *
3720  * controller cleanup handler, Message unit reset or soft reset
3721  * and shutdown notification is issued to the controller and the
3722  * associated memory resources are freed.
3723  *
3724  * Return: Nothing.
3725  */
3726 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc, u8 reason)
3727 {
3728         enum mpi3mr_iocstate ioc_state;
3729
3730         if (reason == MPI3MR_COMPLETE_CLEANUP)
3731                 mpi3mr_stop_watchdog(mrioc);
3732
3733         mpi3mr_ioc_disable_intr(mrioc);
3734
3735         ioc_state = mpi3mr_get_iocstate(mrioc);
3736
3737         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
3738             (ioc_state == MRIOC_STATE_READY)) {
3739                 if (mpi3mr_issue_and_process_mur(mrioc,
3740                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
3741                         mpi3mr_issue_reset(mrioc,
3742                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3743                             MPI3MR_RESET_FROM_MUR_FAILURE);
3744
3745                 if (reason != MPI3MR_REINIT_FAILURE)
3746                         mpi3mr_issue_ioc_shutdown(mrioc);
3747         }
3748
3749         if (reason == MPI3MR_COMPLETE_CLEANUP) {
3750                 mpi3mr_free_mem(mrioc);
3751                 mpi3mr_cleanup_resources(mrioc);
3752         }
3753 }
3754
3755 /**
3756  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
3757  * @mrioc: Adapter instance reference
3758  * @cmdptr: Internal command tracker
3759  *
3760  * Complete an internal driver commands with state indicating it
3761  * is completed due to reset.
3762  *
3763  * Return: Nothing.
3764  */
3765 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
3766         struct mpi3mr_drv_cmd *cmdptr)
3767 {
3768         if (cmdptr->state & MPI3MR_CMD_PENDING) {
3769                 cmdptr->state |= MPI3MR_CMD_RESET;
3770                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
3771                 if (cmdptr->is_waiting) {
3772                         complete(&cmdptr->done);
3773                         cmdptr->is_waiting = 0;
3774                 } else if (cmdptr->callback)
3775                         cmdptr->callback(mrioc, cmdptr);
3776         }
3777 }
3778
3779 /**
3780  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
3781  * @mrioc: Adapter instance reference
3782  *
3783  * Flush all internal driver commands post reset
3784  *
3785  * Return: Nothing.
3786  */
3787 static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
3788 {
3789         struct mpi3mr_drv_cmd *cmdptr;
3790         u8 i;
3791
3792         cmdptr = &mrioc->init_cmds;
3793         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3794         cmdptr = &mrioc->host_tm_cmds;
3795         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3796
3797         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3798                 cmdptr = &mrioc->dev_rmhs_cmds[i];
3799                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3800         }
3801 }
3802
3803 /**
3804  * mpi3mr_diagfault_reset_handler - Diag fault reset handler
3805  * @mrioc: Adapter instance reference
3806  * @reset_reason: Reset reason code
3807  *
3808  * This is an handler for issuing diag fault reset from the
3809  * applications through IOCTL path to stop the execution of the
3810  * controller
3811  *
3812  * Return: 0 on success, non-zero on failure.
3813  */
3814 int mpi3mr_diagfault_reset_handler(struct mpi3mr_ioc *mrioc,
3815         u32 reset_reason)
3816 {
3817         int retval = 0;
3818
3819         ioc_info(mrioc, "Entry: reason code: %s\n",
3820             mpi3mr_reset_rc_name(reset_reason));
3821         mrioc->reset_in_progress = 1;
3822
3823         mpi3mr_ioc_disable_intr(mrioc);
3824
3825         retval = mpi3mr_issue_reset(mrioc,
3826             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3827
3828         if (retval) {
3829                 ioc_err(mrioc, "The diag fault reset failed: reason %d\n",
3830                     reset_reason);
3831                 mpi3mr_ioc_enable_intr(mrioc);
3832         }
3833         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3834         mrioc->reset_in_progress = 0;
3835         return retval;
3836 }
3837
3838 /**
3839  * mpi3mr_soft_reset_handler - Reset the controller
3840  * @mrioc: Adapter instance reference
3841  * @reset_reason: Reset reason code
3842  * @snapdump: Flag to generate snapdump in firmware or not
3843  *
3844  * This is an handler for recovering controller by issuing soft
3845  * reset are diag fault reset.  This is a blocking function and
3846  * when one reset is executed if any other resets they will be
3847  * blocked. All IOCTLs/IO will be blocked during the reset. If
3848  * controller reset is successful then the controller will be
3849  * reinitalized, otherwise the controller will be marked as not
3850  * recoverable
3851  *
3852  * In snapdump bit is set, the controller is issued with diag
3853  * fault reset so that the firmware can create a snap dump and
3854  * post that the firmware will result in F000 fault and the
3855  * driver will issue soft reset to recover from that.
3856  *
3857  * Return: 0 on success, non-zero on failure.
3858  */
3859 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
3860         u32 reset_reason, u8 snapdump)
3861 {
3862         int retval = 0, i;
3863         unsigned long flags;
3864         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
3865
3866         if (mrioc->fault_dbg) {
3867                 if (snapdump)
3868                         mpi3mr_set_diagsave(mrioc);
3869                 mpi3mr_kill_ioc(mrioc, reset_reason);
3870         }
3871
3872         /*
3873          * Block new resets until the currently executing one is finished and
3874          * return the status of the existing reset for all blocked resets
3875          */
3876         if (!mutex_trylock(&mrioc->reset_mutex)) {
3877                 ioc_info(mrioc, "Another reset in progress\n");
3878                 return -1;
3879         }
3880         mrioc->reset_in_progress = 1;
3881
3882         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
3883             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
3884                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3885                         mrioc->event_masks[i] = -1;
3886
3887                 retval = mpi3mr_issue_event_notification(mrioc);
3888
3889                 if (retval) {
3890                         ioc_err(mrioc,
3891                             "Failed to turn off events prior to reset %d\n",
3892                             retval);
3893                 }
3894         }
3895
3896         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
3897
3898         mpi3mr_ioc_disable_intr(mrioc);
3899
3900         if (snapdump) {
3901                 mpi3mr_set_diagsave(mrioc);
3902                 retval = mpi3mr_issue_reset(mrioc,
3903                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3904                 if (!retval) {
3905                         do {
3906                                 host_diagnostic =
3907                                     readl(&mrioc->sysif_regs->host_diagnostic);
3908                                 if (!(host_diagnostic &
3909                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
3910                                         break;
3911                                 msleep(100);
3912                         } while (--timeout);
3913                 }
3914         }
3915
3916         retval = mpi3mr_issue_reset(mrioc,
3917             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
3918         if (retval) {
3919                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
3920                 goto out;
3921         }
3922
3923         mpi3mr_flush_delayed_rmhs_list(mrioc);
3924         mpi3mr_flush_drv_cmds(mrioc);
3925         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3926         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3927         mpi3mr_cleanup_fwevt_list(mrioc);
3928         mpi3mr_flush_host_io(mrioc);
3929         mpi3mr_invalidate_devhandles(mrioc);
3930         mpi3mr_memset_buffers(mrioc);
3931         retval = mpi3mr_init_ioc(mrioc, MPI3MR_IT_RESET);
3932         if (retval) {
3933                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
3934                     mrioc->name, reset_reason);
3935                 goto out;
3936         }
3937         ssleep(10);
3938
3939 out:
3940         if (!retval) {
3941                 mrioc->reset_in_progress = 0;
3942                 scsi_unblock_requests(mrioc->shost);
3943                 mpi3mr_rfresh_tgtdevs(mrioc);
3944                 mrioc->ts_update_counter = 0;
3945                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
3946                 if (mrioc->watchdog_work_q)
3947                         queue_delayed_work(mrioc->watchdog_work_q,
3948                             &mrioc->watchdog_work,
3949                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
3950                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
3951         } else {
3952                 mpi3mr_issue_reset(mrioc,
3953                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3954                 mrioc->unrecoverable = 1;
3955                 mrioc->reset_in_progress = 0;
3956                 retval = -1;
3957         }
3958
3959         mutex_unlock(&mrioc->reset_mutex);
3960         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3961         return retval;
3962 }