drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2022 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 static int
  14 mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
  15 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
  16 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
  17         struct mpi3_ioc_facts_data *facts_data);
  18 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
  19         struct mpi3mr_drv_cmd *drv_cmd);
  20
  21 static int poll_queues;
  22 module_param(poll_queues, int, 0444);
  23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
  24
  25 #if defined(writeq) && defined(CONFIG_64BIT)
  26 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  27 {
  28         writeq(b, addr);
  29 }
  30 #else
  31 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  32 {
  33         __u64 data_out = b;
  34
  35         writel((u32)(data_out), addr);
  36         writel((u32)(data_out >> 32), (addr + 4));
  37 }
  38 #endif
  39
  40 static inline bool
  41 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  42 {
  43         u16 pi, ci, max_entries;
  44         bool is_qfull = false;
  45
  46         pi = op_req_q->pi;
  47         ci = READ_ONCE(op_req_q->ci);
  48         max_entries = op_req_q->num_requests;
  49
  50         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  51                 is_qfull = true;
  52
  53         return is_qfull;
  54 }
  55
  56 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  57 {
  58         u16 i, max_vectors;
  59
  60         max_vectors = mrioc->intr_info_count;
  61
  62         for (i = 0; i < max_vectors; i++)
  63                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  64 }
  65
  66 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  67 {
  68         mrioc->intr_enabled = 0;
  69         mpi3mr_sync_irqs(mrioc);
  70 }
  71
  72 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  73 {
  74         mrioc->intr_enabled = 1;
  75 }
  76
  77 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  78 {
  79         u16 i;
  80
  81         mpi3mr_ioc_disable_intr(mrioc);
  82
  83         if (!mrioc->intr_info)
  84                 return;
  85
  86         for (i = 0; i < mrioc->intr_info_count; i++)
  87                 free_irq(pci_irq_vector(mrioc->pdev, i),
  88                     (mrioc->intr_info + i));
  89
  90         kfree(mrioc->intr_info);
  91         mrioc->intr_info = NULL;
  92         mrioc->intr_info_count = 0;
  93         mrioc->is_intr_info_set = false;
  94         pci_free_irq_vectors(mrioc->pdev);
  95 }
  96
  97 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  98         dma_addr_t dma_addr)
  99 {
 100         struct mpi3_sge_common *sgel = paddr;
 101
 102         sgel->flags = flags;
 103         sgel->length = cpu_to_le32(length);
 104         sgel->address = cpu_to_le64(dma_addr);
 105 }
 106
 107 void mpi3mr_build_zero_len_sge(void *paddr)
 108 {
 109         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
 110
 111         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
 112 }
 113
 114 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         if ((phys_addr < mrioc->reply_buf_dma) ||
 121             (phys_addr > mrioc->reply_buf_dma_max_address))
 122                 return NULL;
 123
 124         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 125 }
 126
 127 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 128         dma_addr_t phys_addr)
 129 {
 130         if (!phys_addr)
 131                 return NULL;
 132
 133         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 134 }
 135
 136 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 137         u64 reply_dma)
 138 {
 139         u32 old_idx = 0;
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 143         old_idx  =  mrioc->reply_free_queue_host_index;
 144         mrioc->reply_free_queue_host_index = (
 145             (mrioc->reply_free_queue_host_index ==
 146             (mrioc->reply_free_qsz - 1)) ? 0 :
 147             (mrioc->reply_free_queue_host_index + 1));
 148         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 149         writel(mrioc->reply_free_queue_host_index,
 150             &mrioc->sysif_regs->reply_free_host_index);
 151         spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 152 }
 153
 154 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 155         u64 sense_buf_dma)
 156 {
 157         u32 old_idx = 0;
 158         unsigned long flags;
 159
 160         spin_lock_irqsave(&mrioc->sbq_lock, flags);
 161         old_idx  =  mrioc->sbq_host_index;
 162         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 163             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 164             (mrioc->sbq_host_index + 1));
 165         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 166         writel(mrioc->sbq_host_index,
 167             &mrioc->sysif_regs->sense_buffer_free_host_index);
 168         spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 169 }
 170
 171 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 172         struct mpi3_event_notification_reply *event_reply)
 173 {
 174         char *desc = NULL;
 175         u16 event;
 176
 177         event = event_reply->event;
 178
 179         switch (event) {
 180         case MPI3_EVENT_LOG_DATA:
 181                 desc = "Log Data";
 182                 break;
 183         case MPI3_EVENT_CHANGE:
 184                 desc = "Event Change";
 185                 break;
 186         case MPI3_EVENT_GPIO_INTERRUPT:
 187                 desc = "GPIO Interrupt";
 188                 break;
 189         case MPI3_EVENT_CABLE_MGMT:
 190                 desc = "Cable Management";
 191                 break;
 192         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 193                 desc = "Energy Pack Change";
 194                 break;
 195         case MPI3_EVENT_DEVICE_ADDED:
 196         {
 197                 struct mpi3_device_page0 *event_data =
 198                     (struct mpi3_device_page0 *)event_reply->event_data;
 199                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 200                     event_data->dev_handle, event_data->device_form);
 201                 return;
 202         }
 203         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 204         {
 205                 struct mpi3_device_page0 *event_data =
 206                     (struct mpi3_device_page0 *)event_reply->event_data;
 207                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 208                     event_data->dev_handle, event_data->device_form);
 209                 return;
 210         }
 211         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 212         {
 213                 struct mpi3_event_data_device_status_change *event_data =
 214                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 215                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 216                     event_data->dev_handle, event_data->reason_code);
 217                 return;
 218         }
 219         case MPI3_EVENT_SAS_DISCOVERY:
 220         {
 221                 struct mpi3_event_data_sas_discovery *event_data =
 222                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 223                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 224                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 225                     "start" : "stop",
 226                     le32_to_cpu(event_data->discovery_status));
 227                 return;
 228         }
 229         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 230                 desc = "SAS Broadcast Primitive";
 231                 break;
 232         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 233                 desc = "SAS Notify Primitive";
 234                 break;
 235         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 236                 desc = "SAS Init Device Status Change";
 237                 break;
 238         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 239                 desc = "SAS Init Table Overflow";
 240                 break;
 241         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 242                 desc = "SAS Topology Change List";
 243                 break;
 244         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 245                 desc = "Enclosure Device Status Change";
 246                 break;
 247         case MPI3_EVENT_HARD_RESET_RECEIVED:
 248                 desc = "Hard Reset Received";
 249                 break;
 250         case MPI3_EVENT_SAS_PHY_COUNTER:
 251                 desc = "SAS PHY Counter";
 252                 break;
 253         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 254                 desc = "SAS Device Discovery Error";
 255                 break;
 256         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 257                 desc = "PCIE Topology Change List";
 258                 break;
 259         case MPI3_EVENT_PCIE_ENUMERATION:
 260         {
 261                 struct mpi3_event_data_pcie_enumeration *event_data =
 262                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 263                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 264                     (event_data->reason_code ==
 265                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 266                 if (event_data->enumeration_status)
 267                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 268                             le32_to_cpu(event_data->enumeration_status));
 269                 return;
 270         }
 271         case MPI3_EVENT_PREPARE_FOR_RESET:
 272                 desc = "Prepare For Reset";
 273                 break;
 274         }
 275
 276         if (!desc)
 277                 return;
 278
 279         ioc_info(mrioc, "%s\n", desc);
 280 }
 281
 282 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 283         struct mpi3_default_reply *def_reply)
 284 {
 285         struct mpi3_event_notification_reply *event_reply =
 286             (struct mpi3_event_notification_reply *)def_reply;
 287
 288         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 289         mpi3mr_print_event_data(mrioc, event_reply);
 290         mpi3mr_os_handle_events(mrioc, event_reply);
 291 }
 292
 293 static struct mpi3mr_drv_cmd *
 294 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 295         struct mpi3_default_reply *def_reply)
 296 {
 297         u16 idx;
 298
 299         switch (host_tag) {
 300         case MPI3MR_HOSTTAG_INITCMDS:
 301                 return &mrioc->init_cmds;
 302         case MPI3MR_HOSTTAG_BSG_CMDS:
 303                 return &mrioc->bsg_cmds;
 304         case MPI3MR_HOSTTAG_BLK_TMS:
 305                 return &mrioc->host_tm_cmds;
 306         case MPI3MR_HOSTTAG_PEL_ABORT:
 307                 return &mrioc->pel_abort_cmd;
 308         case MPI3MR_HOSTTAG_PEL_WAIT:
 309                 return &mrioc->pel_cmds;
 310         case MPI3MR_HOSTTAG_INVALID:
 311                 if (def_reply && def_reply->function ==
 312                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 313                         mpi3mr_handle_events(mrioc, def_reply);
 314                 return NULL;
 315         default:
 316                 break;
 317         }
 318         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 319             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 320                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 321                 return &mrioc->dev_rmhs_cmds[idx];
 322         }
 323
 324         if (host_tag >= MPI3MR_HOSTTAG_EVTACKCMD_MIN &&
 325             host_tag <= MPI3MR_HOSTTAG_EVTACKCMD_MAX) {
 326                 idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
 327                 return &mrioc->evtack_cmds[idx];
 328         }
 329
 330         return NULL;
 331 }
 332
 333 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 334         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 335 {
 336         u16 reply_desc_type, host_tag = 0;
 337         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 338         u32 ioc_loginfo = 0;
 339         struct mpi3_status_reply_descriptor *status_desc;
 340         struct mpi3_address_reply_descriptor *addr_desc;
 341         struct mpi3_success_reply_descriptor *success_desc;
 342         struct mpi3_default_reply *def_reply = NULL;
 343         struct mpi3mr_drv_cmd *cmdptr = NULL;
 344         struct mpi3_scsi_io_reply *scsi_reply;
 345         u8 *sense_buf = NULL;
 346
 347         *reply_dma = 0;
 348         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 349             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 350         switch (reply_desc_type) {
 351         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 352                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 353                 host_tag = le16_to_cpu(status_desc->host_tag);
 354                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 355                 if (ioc_status &
 356                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 357                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 358                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 359                 break;
 360         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 361                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 362                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 363                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 364                 if (!def_reply)
 365                         goto out;
 366                 host_tag = le16_to_cpu(def_reply->host_tag);
 367                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 368                 if (ioc_status &
 369                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 370                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 371                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 372                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 373                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 374                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 375                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 376                 }
 377                 break;
 378         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 379                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 380                 host_tag = le16_to_cpu(success_desc->host_tag);
 381                 break;
 382         default:
 383                 break;
 384         }
 385
 386         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 387         if (cmdptr) {
 388                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 389                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 390                         cmdptr->ioc_loginfo = ioc_loginfo;
 391                         cmdptr->ioc_status = ioc_status;
 392                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 393                         if (def_reply) {
 394                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 395                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 396                                     mrioc->reply_sz);
 397                         }
 398                         if (cmdptr->is_waiting) {
 399                                 complete(&cmdptr->done);
 400                                 cmdptr->is_waiting = 0;
 401                         } else if (cmdptr->callback)
 402                                 cmdptr->callback(mrioc, cmdptr);
 403                 }
 404         }
 405 out:
 406         if (sense_buf)
 407                 mpi3mr_repost_sense_buf(mrioc,
 408                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 409 }
 410
 411 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 412 {
 413         u32 exp_phase = mrioc->admin_reply_ephase;
 414         u32 admin_reply_ci = mrioc->admin_reply_ci;
 415         u32 num_admin_replies = 0;
 416         u64 reply_dma = 0;
 417         struct mpi3_default_reply_descriptor *reply_desc;
 418
 419         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 420             admin_reply_ci;
 421
 422         if ((le16_to_cpu(reply_desc->reply_flags) &
 423             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 424                 return 0;
 425
 426         do {
 427                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 428                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 429                 if (reply_dma)
 430                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 431                 num_admin_replies++;
 432                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 433                         admin_reply_ci = 0;
 434                         exp_phase ^= 1;
 435                 }
 436                 reply_desc =
 437                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 438                     admin_reply_ci;
 439                 if ((le16_to_cpu(reply_desc->reply_flags) &
 440                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 441                         break;
 442         } while (1);
 443
 444         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 445         mrioc->admin_reply_ci = admin_reply_ci;
 446         mrioc->admin_reply_ephase = exp_phase;
 447
 448         return num_admin_replies;
 449 }
 450
 451 /**
 452  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 453  *      queue's consumer index from operational reply descriptor queue.
 454  * @op_reply_q: op_reply_qinfo object
 455  * @reply_ci: operational reply descriptor's queue consumer index
 456  *
 457  * Returns reply descriptor frame address
 458  */
 459 static inline struct mpi3_default_reply_descriptor *
 460 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 461 {
 462         void *segment_base_addr;
 463         struct segments *segments = op_reply_q->q_segments;
 464         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 465
 466         segment_base_addr =
 467             segments[reply_ci / op_reply_q->segment_qd].segment;
 468         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 469             (reply_ci % op_reply_q->segment_qd);
 470         return reply_desc;
 471 }
 472
 473 /**
 474  * mpi3mr_process_op_reply_q - Operational reply queue handler
 475  * @mrioc: Adapter instance reference
 476  * @op_reply_q: Operational reply queue info
 477  *
 478  * Checks the specific operational reply queue and drains the
 479  * reply queue entries until the queue is empty and process the
 480  * individual reply descriptors.
 481  *
 482  * Return: 0 if queue is already processed,or number of reply
 483  *          descriptors processed.
 484  */
 485 int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 486         struct op_reply_qinfo *op_reply_q)
 487 {
 488         struct op_req_qinfo *op_req_q;
 489         u32 exp_phase;
 490         u32 reply_ci;
 491         u32 num_op_reply = 0;
 492         u64 reply_dma = 0;
 493         struct mpi3_default_reply_descriptor *reply_desc;
 494         u16 req_q_idx = 0, reply_qidx;
 495
 496         reply_qidx = op_reply_q->qid - 1;
 497
 498         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 499                 return 0;
 500
 501         exp_phase = op_reply_q->ephase;
 502         reply_ci = op_reply_q->ci;
 503
 504         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 505         if ((le16_to_cpu(reply_desc->reply_flags) &
 506             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 507                 atomic_dec(&op_reply_q->in_use);
 508                 return 0;
 509         }
 510
 511         do {
 512                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 513                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 514
 515                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 516                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 517                     reply_qidx);
 518                 atomic_dec(&op_reply_q->pend_ios);
 519                 if (reply_dma)
 520                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 521                 num_op_reply++;
 522
 523                 if (++reply_ci == op_reply_q->num_replies) {
 524                         reply_ci = 0;
 525                         exp_phase ^= 1;
 526                 }
 527
 528                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 529
 530                 if ((le16_to_cpu(reply_desc->reply_flags) &
 531                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 532                         break;
 533                 /*
 534                  * Exit completion loop to avoid CPU lockup
 535                  * Ensure remaining completion happens from threaded ISR.
 536                  */
 537                 if (num_op_reply > mrioc->max_host_ios) {
 538                         op_reply_q->enable_irq_poll = true;
 539                         break;
 540                 }
 541
 542         } while (1);
 543
 544         writel(reply_ci,
 545             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 546         op_reply_q->ci = reply_ci;
 547         op_reply_q->ephase = exp_phase;
 548
 549         atomic_dec(&op_reply_q->in_use);
 550         return num_op_reply;
 551 }
 552
 553 /**
 554  * mpi3mr_blk_mq_poll - Operational reply queue handler
 555  * @shost: SCSI Host reference
 556  * @queue_num: Request queue number (w.r.t OS it is hardware context number)
 557  *
 558  * Checks the specific operational reply queue and drains the
 559  * reply queue entries until the queue is empty and process the
 560  * individual reply descriptors.
 561  *
 562  * Return: 0 if queue is already processed,or number of reply
 563  *          descriptors processed.
 564  */
 565 int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
 566 {
 567         int num_entries = 0;
 568         struct mpi3mr_ioc *mrioc;
 569
 570         mrioc = (struct mpi3mr_ioc *)shost->hostdata;
 571
 572         if ((mrioc->reset_in_progress || mrioc->prepare_for_reset))
 573                 return 0;
 574
 575         num_entries = mpi3mr_process_op_reply_q(mrioc,
 576                         &mrioc->op_reply_qinfo[queue_num]);
 577
 578         return num_entries;
 579 }
 580
 581 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 582 {
 583         struct mpi3mr_intr_info *intr_info = privdata;
 584         struct mpi3mr_ioc *mrioc;
 585         u16 midx;
 586         u32 num_admin_replies = 0, num_op_reply = 0;
 587
 588         if (!intr_info)
 589                 return IRQ_NONE;
 590
 591         mrioc = intr_info->mrioc;
 592
 593         if (!mrioc->intr_enabled)
 594                 return IRQ_NONE;
 595
 596         midx = intr_info->msix_index;
 597
 598         if (!midx)
 599                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 600         if (intr_info->op_reply_q)
 601                 num_op_reply = mpi3mr_process_op_reply_q(mrioc,
 602                     intr_info->op_reply_q);
 603
 604         if (num_admin_replies || num_op_reply)
 605                 return IRQ_HANDLED;
 606         else
 607                 return IRQ_NONE;
 608 }
 609
 610 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 611 {
 612         struct mpi3mr_intr_info *intr_info = privdata;
 613         struct mpi3mr_ioc *mrioc;
 614         u16 midx;
 615         int ret;
 616
 617         if (!intr_info)
 618                 return IRQ_NONE;
 619
 620         mrioc = intr_info->mrioc;
 621         midx = intr_info->msix_index;
 622         /* Call primary ISR routine */
 623         ret = mpi3mr_isr_primary(irq, privdata);
 624
 625         /*
 626          * If more IOs are expected, schedule IRQ polling thread.
 627          * Otherwise exit from ISR.
 628          */
 629         if (!intr_info->op_reply_q)
 630                 return ret;
 631
 632         if (!intr_info->op_reply_q->enable_irq_poll ||
 633             !atomic_read(&intr_info->op_reply_q->pend_ios))
 634                 return ret;
 635
 636         disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx));
 637
 638         return IRQ_WAKE_THREAD;
 639 }
 640
 641 /**
 642  * mpi3mr_isr_poll - Reply queue polling routine
 643  * @irq: IRQ
 644  * @privdata: Interrupt info
 645  *
 646  * poll for pending I/O completions in a loop until pending I/Os
 647  * present or controller queue depth I/Os are processed.
 648  *
 649  * Return: IRQ_NONE or IRQ_HANDLED
 650  */
 651 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 652 {
 653         struct mpi3mr_intr_info *intr_info = privdata;
 654         struct mpi3mr_ioc *mrioc;
 655         u16 midx;
 656         u32 num_op_reply = 0;
 657
 658         if (!intr_info || !intr_info->op_reply_q)
 659                 return IRQ_NONE;
 660
 661         mrioc = intr_info->mrioc;
 662         midx = intr_info->msix_index;
 663
 664         /* Poll for pending IOs completions */
 665         do {
 666                 if (!mrioc->intr_enabled)
 667                         break;
 668
 669                 if (!midx)
 670                         mpi3mr_process_admin_reply_q(mrioc);
 671                 if (intr_info->op_reply_q)
 672                         num_op_reply +=
 673                             mpi3mr_process_op_reply_q(mrioc,
 674                                 intr_info->op_reply_q);
 675
 676                 usleep_range(MPI3MR_IRQ_POLL_SLEEP, 10 * MPI3MR_IRQ_POLL_SLEEP);
 677
 678         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 679             (num_op_reply < mrioc->max_host_ios));
 680
 681         intr_info->op_reply_q->enable_irq_poll = false;
 682         enable_irq(pci_irq_vector(mrioc->pdev, midx));
 683
 684         return IRQ_HANDLED;
 685 }
 686
 687 /**
 688  * mpi3mr_request_irq - Request IRQ and register ISR
 689  * @mrioc: Adapter instance reference
 690  * @index: IRQ vector index
 691  *
 692  * Request threaded ISR with primary ISR and secondary
 693  *
 694  * Return: 0 on success and non zero on failures.
 695  */
 696 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 697 {
 698         struct pci_dev *pdev = mrioc->pdev;
 699         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 700         int retval = 0;
 701
 702         intr_info->mrioc = mrioc;
 703         intr_info->msix_index = index;
 704         intr_info->op_reply_q = NULL;
 705
 706         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 707             mrioc->driver_name, mrioc->id, index);
 708
 709         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 710             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 711         if (retval) {
 712                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 713                     intr_info->name, pci_irq_vector(pdev, index));
 714                 return retval;
 715         }
 716
 717         return retval;
 718 }
 719
 720 static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
 721 {
 722         if (!mrioc->requested_poll_qcount)
 723                 return;
 724
 725         /* Reserved for Admin and Default Queue */
 726         if (max_vectors > 2 &&
 727                 (mrioc->requested_poll_qcount < max_vectors - 2)) {
 728                 ioc_info(mrioc,
 729                     "enabled polled queues (%d) msix (%d)\n",
 730                     mrioc->requested_poll_qcount, max_vectors);
 731         } else {
 732                 ioc_info(mrioc,
 733                     "disabled polled queues (%d) msix (%d) because of no resources for default queue\n",
 734                     mrioc->requested_poll_qcount, max_vectors);
 735                 mrioc->requested_poll_qcount = 0;
 736         }
 737 }
 738
 739 /**
 740  * mpi3mr_setup_isr - Setup ISR for the controller
 741  * @mrioc: Adapter instance reference
 742  * @setup_one: Request one IRQ or more
 743  *
 744  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 745  *
 746  * Return: 0 on success and non zero on failures.
 747  */
 748 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 749 {
 750         unsigned int irq_flags = PCI_IRQ_MSIX;
 751         int max_vectors, min_vec;
 752         int retval;
 753         int i;
 754         struct irq_affinity desc = { .pre_vectors =  1, .post_vectors = 1 };
 755
 756         if (mrioc->is_intr_info_set)
 757                 return 0;
 758
 759         mpi3mr_cleanup_isr(mrioc);
 760
 761         if (setup_one || reset_devices) {
 762                 max_vectors = 1;
 763                 retval = pci_alloc_irq_vectors(mrioc->pdev,
 764                     1, max_vectors, irq_flags);
 765                 if (retval < 0) {
 766                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 767                             retval);
 768                         goto out_failed;
 769                 }
 770         } else {
 771                 max_vectors =
 772                     min_t(int, mrioc->cpu_count + 1 +
 773                         mrioc->requested_poll_qcount, mrioc->msix_count);
 774
 775                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 776
 777                 ioc_info(mrioc,
 778                     "MSI-X vectors supported: %d, no of cores: %d,",
 779                     mrioc->msix_count, mrioc->cpu_count);
 780                 ioc_info(mrioc,
 781                     "MSI-x vectors requested: %d poll_queues %d\n",
 782                     max_vectors, mrioc->requested_poll_qcount);
 783
 784                 desc.post_vectors = mrioc->requested_poll_qcount;
 785                 min_vec = desc.pre_vectors + desc.post_vectors;
 786                 irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 787
 788                 retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 789                         min_vec, max_vectors, irq_flags, &desc);
 790
 791                 if (retval < 0) {
 792                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 793                             retval);
 794                         goto out_failed;
 795                 }
 796
 797
 798                 /*
 799                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 800                  * between Admin queue and operational queue
 801                  */
 802                 if (retval == min_vec)
 803                         mrioc->op_reply_q_offset = 0;
 804                 else if (retval != (max_vectors)) {
 805                         ioc_info(mrioc,
 806                             "allocated vectors (%d) are less than configured (%d)\n",
 807                             retval, max_vectors);
 808                 }
 809
 810                 max_vectors = retval;
 811                 mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 812
 813                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 814
 815         }
 816
 817         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 818             GFP_KERNEL);
 819         if (!mrioc->intr_info) {
 820                 retval = -ENOMEM;
 821                 pci_free_irq_vectors(mrioc->pdev);
 822                 goto out_failed;
 823         }
 824         for (i = 0; i < max_vectors; i++) {
 825                 retval = mpi3mr_request_irq(mrioc, i);
 826                 if (retval) {
 827                         mrioc->intr_info_count = i;
 828                         goto out_failed;
 829                 }
 830         }
 831         if (reset_devices || !setup_one)
 832                 mrioc->is_intr_info_set = true;
 833         mrioc->intr_info_count = max_vectors;
 834         mpi3mr_ioc_enable_intr(mrioc);
 835         return 0;
 836
 837 out_failed:
 838         mpi3mr_cleanup_isr(mrioc);
 839
 840         return retval;
 841 }
 842
 843 static const struct {
 844         enum mpi3mr_iocstate value;
 845         char *name;
 846 } mrioc_states[] = {
 847         { MRIOC_STATE_READY, "ready" },
 848         { MRIOC_STATE_FAULT, "fault" },
 849         { MRIOC_STATE_RESET, "reset" },
 850         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 851         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 852         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 853 };
 854
 855 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 856 {
 857         int i;
 858         char *name = NULL;
 859
 860         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 861                 if (mrioc_states[i].value == mrioc_state) {
 862                         name = mrioc_states[i].name;
 863                         break;
 864                 }
 865         }
 866         return name;
 867 }
 868
 869 /* Reset reason to name mapper structure*/
 870 static const struct {
 871         enum mpi3mr_reset_reason value;
 872         char *name;
 873 } mpi3mr_reset_reason_codes[] = {
 874         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 875         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 876         { MPI3MR_RESET_FROM_APP, "application invocation" },
 877         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 878         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 879         { MPI3MR_RESET_FROM_APP_TIMEOUT, "application command timeout" },
 880         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 881         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 882         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 883         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 884         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 885         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 886         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 887         {
 888                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 889                 "create request queue timeout"
 890         },
 891         {
 892                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 893                 "create reply queue timeout"
 894         },
 895         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 896         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 897         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 898         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 899         {
 900                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 901                 "component image activation timeout"
 902         },
 903         {
 904                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 905                 "get package version timeout"
 906         },
 907         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 908         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 909         { MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
 910 };
 911
 912 /**
 913  * mpi3mr_reset_rc_name - get reset reason code name
 914  * @reason_code: reset reason code value
 915  *
 916  * Map reset reason to an NULL terminated ASCII string
 917  *
 918  * Return: name corresponding to reset reason value or NULL.
 919  */
 920 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 921 {
 922         int i;
 923         char *name = NULL;
 924
 925         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 926                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 927                         name = mpi3mr_reset_reason_codes[i].name;
 928                         break;
 929                 }
 930         }
 931         return name;
 932 }
 933
 934 /* Reset type to name mapper structure*/
 935 static const struct {
 936         u16 reset_type;
 937         char *name;
 938 } mpi3mr_reset_types[] = {
 939         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 940         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 941 };
 942
 943 /**
 944  * mpi3mr_reset_type_name - get reset type name
 945  * @reset_type: reset type value
 946  *
 947  * Map reset type to an NULL terminated ASCII string
 948  *
 949  * Return: name corresponding to reset type value or NULL.
 950  */
 951 static const char *mpi3mr_reset_type_name(u16 reset_type)
 952 {
 953         int i;
 954         char *name = NULL;
 955
 956         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 957                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 958                         name = mpi3mr_reset_types[i].name;
 959                         break;
 960                 }
 961         }
 962         return name;
 963 }
 964
 965 /**
 966  * mpi3mr_print_fault_info - Display fault information
 967  * @mrioc: Adapter instance reference
 968  *
 969  * Display the controller fault information if there is a
 970  * controller fault.
 971  *
 972  * Return: Nothing.
 973  */
 974 void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 975 {
 976         u32 ioc_status, code, code1, code2, code3;
 977
 978         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 979
 980         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
 981                 code = readl(&mrioc->sysif_regs->fault);
 982                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
 983                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
 984                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
 985
 986                 ioc_info(mrioc,
 987                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
 988                     code, code1, code2, code3);
 989         }
 990 }
 991
 992 /**
 993  * mpi3mr_get_iocstate - Get IOC State
 994  * @mrioc: Adapter instance reference
 995  *
 996  * Return a proper IOC state enum based on the IOC status and
 997  * IOC configuration and unrcoverable state of the controller.
 998  *
 999  * Return: Current IOC state.
1000  */
1001 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1002 {
1003         u32 ioc_status, ioc_config;
1004         u8 ready, enabled;
1005
1006         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1007         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1008
1009         if (mrioc->unrecoverable)
1010                 return MRIOC_STATE_UNRECOVERABLE;
1011         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
1012                 return MRIOC_STATE_FAULT;
1013
1014         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
1015         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
1016
1017         if (ready && enabled)
1018                 return MRIOC_STATE_READY;
1019         if ((!ready) && (!enabled))
1020                 return MRIOC_STATE_RESET;
1021         if ((!ready) && (enabled))
1022                 return MRIOC_STATE_BECOMING_READY;
1023
1024         return MRIOC_STATE_RESET_REQUESTED;
1025 }
1026
1027 /**
1028  * mpi3mr_clear_reset_history - clear reset history
1029  * @mrioc: Adapter instance reference
1030  *
1031  * Write the reset history bit in IOC status to clear the bit,
1032  * if it is already set.
1033  *
1034  * Return: Nothing.
1035  */
1036 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1037 {
1038         u32 ioc_status;
1039
1040         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1041         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1042                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
1043 }
1044
1045 /**
1046  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1047  * @mrioc: Adapter instance reference
1048  * @reset_reason: Reset reason code
1049  *
1050  * Issue Message unit Reset to the controller and wait for it to
1051  * be complete.
1052  *
1053  * Return: 0 on success, -1 on failure.
1054  */
1055 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1056         u32 reset_reason)
1057 {
1058         u32 ioc_config, timeout, ioc_status;
1059         int retval = -1;
1060
1061         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1062         if (mrioc->unrecoverable) {
1063                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1064                 return retval;
1065         }
1066         mpi3mr_clear_reset_history(mrioc);
1067         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1068         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1069         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1070         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1071
1072         timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1073         do {
1074                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1075                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
1076                         mpi3mr_clear_reset_history(mrioc);
1077                         break;
1078                 }
1079                 if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1080                         mpi3mr_print_fault_info(mrioc);
1081                         break;
1082                 }
1083                 msleep(100);
1084         } while (--timeout);
1085
1086         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1087         if (timeout && !((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1088               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1089               (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1090                 retval = 0;
1091
1092         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1093             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
1094         return retval;
1095 }
1096
1097 /**
1098  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1099  * during reset/resume
1100  * @mrioc: Adapter instance reference
1101  *
1102  * Return zero if the new IOCFacts parameters value is compatible with
1103  * older values else return -EPERM
1104  */
1105 static int
1106 mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1107 {
1108         u16 dev_handle_bitmap_sz;
1109         void *removepend_bitmap;
1110
1111         if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1112                 ioc_err(mrioc,
1113                     "cannot increase reply size from %d to %d\n",
1114                     mrioc->reply_sz, mrioc->facts.reply_sz);
1115                 return -EPERM;
1116         }
1117
1118         if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1119                 ioc_err(mrioc,
1120                     "cannot reduce number of operational reply queues from %d to %d\n",
1121                     mrioc->num_op_reply_q,
1122                     mrioc->facts.max_op_reply_q);
1123                 return -EPERM;
1124         }
1125
1126         if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1127                 ioc_err(mrioc,
1128                     "cannot reduce number of operational request queues from %d to %d\n",
1129                     mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1130                 return -EPERM;
1131         }
1132
1133         dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
1134         if (mrioc->facts.max_devhandle % 8)
1135                 dev_handle_bitmap_sz++;
1136         if (dev_handle_bitmap_sz > mrioc->dev_handle_bitmap_sz) {
1137                 removepend_bitmap = krealloc(mrioc->removepend_bitmap,
1138                     dev_handle_bitmap_sz, GFP_KERNEL);
1139                 if (!removepend_bitmap) {
1140                         ioc_err(mrioc,
1141                             "failed to increase removepend_bitmap sz from: %d to %d\n",
1142                             mrioc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);
1143                         return -EPERM;
1144                 }
1145                 memset(removepend_bitmap + mrioc->dev_handle_bitmap_sz, 0,
1146                     dev_handle_bitmap_sz - mrioc->dev_handle_bitmap_sz);
1147                 mrioc->removepend_bitmap = removepend_bitmap;
1148                 ioc_info(mrioc,
1149                     "increased dev_handle_bitmap_sz from %d to %d\n",
1150                     mrioc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);
1151                 mrioc->dev_handle_bitmap_sz = dev_handle_bitmap_sz;
1152         }
1153
1154         return 0;
1155 }
1156
1157 /**
1158  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1159  * @mrioc: Adapter instance reference
1160  *
1161  * Set Enable IOC bit in IOC configuration register and wait for
1162  * the controller to become ready.
1163  *
1164  * Return: 0 on success, appropriate error on failure.
1165  */
1166 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1167 {
1168         u32 ioc_config, ioc_status, timeout;
1169         int retval = 0;
1170         enum mpi3mr_iocstate ioc_state;
1171         u64 base_info;
1172
1173         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1174         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1175         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
1176         ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1177             ioc_status, ioc_config, base_info);
1178
1179         /*The timeout value is in 2sec unit, changing it to seconds*/
1180         mrioc->ready_timeout =
1181             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
1182             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
1183
1184         ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1185
1186         ioc_state = mpi3mr_get_iocstate(mrioc);
1187         ioc_info(mrioc, "controller is in %s state during detection\n",
1188             mpi3mr_iocstate_name(ioc_state));
1189
1190         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
1191             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
1192                 timeout = mrioc->ready_timeout * 10;
1193                 do {
1194                         msleep(100);
1195                 } while (--timeout);
1196
1197                 ioc_state = mpi3mr_get_iocstate(mrioc);
1198                 ioc_info(mrioc,
1199                     "controller is in %s state after waiting to reset\n",
1200                     mpi3mr_iocstate_name(ioc_state));
1201         }
1202
1203         if (ioc_state == MRIOC_STATE_READY) {
1204                 ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1205                 retval = mpi3mr_issue_and_process_mur(mrioc,
1206                     MPI3MR_RESET_FROM_BRINGUP);
1207                 ioc_state = mpi3mr_get_iocstate(mrioc);
1208                 if (retval)
1209                         ioc_err(mrioc,
1210                             "message unit reset failed with error %d current state %s\n",
1211                             retval, mpi3mr_iocstate_name(ioc_state));
1212         }
1213         if (ioc_state != MRIOC_STATE_RESET) {
1214                 mpi3mr_print_fault_info(mrioc);
1215                 ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1216                 retval = mpi3mr_issue_reset(mrioc,
1217                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
1218                     MPI3MR_RESET_FROM_BRINGUP);
1219                 if (retval) {
1220                         ioc_err(mrioc,
1221                             "soft reset failed with error %d\n", retval);
1222                         goto out_failed;
1223                 }
1224         }
1225         ioc_state = mpi3mr_get_iocstate(mrioc);
1226         if (ioc_state != MRIOC_STATE_RESET) {
1227                 ioc_err(mrioc,
1228                     "cannot bring controller to reset state, current state: %s\n",
1229                     mpi3mr_iocstate_name(ioc_state));
1230                 goto out_failed;
1231         }
1232         mpi3mr_clear_reset_history(mrioc);
1233         retval = mpi3mr_setup_admin_qpair(mrioc);
1234         if (retval) {
1235                 ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1236                     retval);
1237                 goto out_failed;
1238         }
1239
1240         ioc_info(mrioc, "bringing controller to ready state\n");
1241         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1242         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1243         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1244
1245         timeout = mrioc->ready_timeout * 10;
1246         do {
1247                 ioc_state = mpi3mr_get_iocstate(mrioc);
1248                 if (ioc_state == MRIOC_STATE_READY) {
1249                         ioc_info(mrioc,
1250                             "successfully transitioned to %s state\n",
1251                             mpi3mr_iocstate_name(ioc_state));
1252                         return 0;
1253                 }
1254                 msleep(100);
1255         } while (--timeout);
1256
1257 out_failed:
1258         ioc_state = mpi3mr_get_iocstate(mrioc);
1259         ioc_err(mrioc,
1260             "failed to bring to ready state,  current state: %s\n",
1261             mpi3mr_iocstate_name(ioc_state));
1262         return retval;
1263 }
1264
1265 /**
1266  * mpi3mr_soft_reset_success - Check softreset is success or not
1267  * @ioc_status: IOC status register value
1268  * @ioc_config: IOC config register value
1269  *
1270  * Check whether the soft reset is successful or not based on
1271  * IOC status and IOC config register values.
1272  *
1273  * Return: True when the soft reset is success, false otherwise.
1274  */
1275 static inline bool
1276 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1277 {
1278         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1279             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1280                 return true;
1281         return false;
1282 }
1283
1284 /**
1285  * mpi3mr_diagfault_success - Check diag fault is success or not
1286  * @mrioc: Adapter reference
1287  * @ioc_status: IOC status register value
1288  *
1289  * Check whether the controller hit diag reset fault code.
1290  *
1291  * Return: True when there is diag fault, false otherwise.
1292  */
1293 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1294         u32 ioc_status)
1295 {
1296         u32 fault;
1297
1298         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1299                 return false;
1300         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1301         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) {
1302                 mpi3mr_print_fault_info(mrioc);
1303                 return true;
1304         }
1305         return false;
1306 }
1307
1308 /**
1309  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1310  * @mrioc: Adapter reference
1311  *
1312  * Set diag save bit in IOC configuration register to enable
1313  * snapdump.
1314  *
1315  * Return: Nothing.
1316  */
1317 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1318 {
1319         u32 ioc_config;
1320
1321         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1322         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1323         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1324 }
1325
1326 /**
1327  * mpi3mr_issue_reset - Issue reset to the controller
1328  * @mrioc: Adapter reference
1329  * @reset_type: Reset type
1330  * @reset_reason: Reset reason code
1331  *
1332  * Unlock the host diagnostic registers and write the specific
1333  * reset type to that, wait for reset acknowledgment from the
1334  * controller, if the reset is not successful retry for the
1335  * predefined number of times.
1336  *
1337  * Return: 0 on success, non-zero on failure.
1338  */
1339 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1340         u32 reset_reason)
1341 {
1342         int retval = -1;
1343         u8 unlock_retry_count = 0;
1344         u32 host_diagnostic, ioc_status, ioc_config;
1345         u32 timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1346
1347         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1348             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1349                 return retval;
1350         if (mrioc->unrecoverable)
1351                 return retval;
1352         if (reset_reason == MPI3MR_RESET_FROM_FIRMWARE) {
1353                 retval = 0;
1354                 return retval;
1355         }
1356
1357         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1358             mpi3mr_reset_type_name(reset_type),
1359             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1360
1361         mpi3mr_clear_reset_history(mrioc);
1362         do {
1363                 ioc_info(mrioc,
1364                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1365                     ++unlock_retry_count);
1366                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1367                         ioc_err(mrioc,
1368                             "%s reset failed due to unlock failure, host_diagnostic(0x%08x)\n",
1369                             mpi3mr_reset_type_name(reset_type),
1370                             host_diagnostic);
1371                         mrioc->unrecoverable = 1;
1372                         return retval;
1373                 }
1374
1375                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1376                     &mrioc->sysif_regs->write_sequence);
1377                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1378                     &mrioc->sysif_regs->write_sequence);
1379                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1380                     &mrioc->sysif_regs->write_sequence);
1381                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1382                     &mrioc->sysif_regs->write_sequence);
1383                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1384                     &mrioc->sysif_regs->write_sequence);
1385                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1386                     &mrioc->sysif_regs->write_sequence);
1387                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1388                     &mrioc->sysif_regs->write_sequence);
1389                 usleep_range(1000, 1100);
1390                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1391                 ioc_info(mrioc,
1392                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1393                     unlock_retry_count, host_diagnostic);
1394         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1395
1396         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1397         writel(host_diagnostic | reset_type,
1398             &mrioc->sysif_regs->host_diagnostic);
1399         switch (reset_type) {
1400         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET:
1401                 do {
1402                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1403                         ioc_config =
1404                             readl(&mrioc->sysif_regs->ioc_configuration);
1405                         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1406                             && mpi3mr_soft_reset_success(ioc_status, ioc_config)
1407                             ) {
1408                                 mpi3mr_clear_reset_history(mrioc);
1409                                 retval = 0;
1410                                 break;
1411                         }
1412                         msleep(100);
1413                 } while (--timeout);
1414                 mpi3mr_print_fault_info(mrioc);
1415                 break;
1416         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT:
1417                 do {
1418                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1419                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1420                                 retval = 0;
1421                                 break;
1422                         }
1423                         msleep(100);
1424                 } while (--timeout);
1425                 break;
1426         default:
1427                 break;
1428         }
1429
1430         writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1431             &mrioc->sysif_regs->write_sequence);
1432
1433         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1434         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1435         ioc_info(mrioc,
1436             "ioc_status/ioc_onfig after %s reset is (0x%x)/(0x%x)\n",
1437             (!retval)?"successful":"failed", ioc_status,
1438             ioc_config);
1439         if (retval)
1440                 mrioc->unrecoverable = 1;
1441         return retval;
1442 }
1443
1444 /**
1445  * mpi3mr_admin_request_post - Post request to admin queue
1446  * @mrioc: Adapter reference
1447  * @admin_req: MPI3 request
1448  * @admin_req_sz: Request size
1449  * @ignore_reset: Ignore reset in process
1450  *
1451  * Post the MPI3 request into admin request queue and
1452  * inform the controller, if the queue is full return
1453  * appropriate error.
1454  *
1455  * Return: 0 on success, non-zero on failure.
1456  */
1457 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1458         u16 admin_req_sz, u8 ignore_reset)
1459 {
1460         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1461         int retval = 0;
1462         unsigned long flags;
1463         u8 *areq_entry;
1464
1465         if (mrioc->unrecoverable) {
1466                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1467                 return -EFAULT;
1468         }
1469
1470         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1471         areq_pi = mrioc->admin_req_pi;
1472         areq_ci = mrioc->admin_req_ci;
1473         max_entries = mrioc->num_admin_req;
1474         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1475             (areq_pi == (max_entries - 1)))) {
1476                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1477                 retval = -EAGAIN;
1478                 goto out;
1479         }
1480         if (!ignore_reset && mrioc->reset_in_progress) {
1481                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1482                 retval = -EAGAIN;
1483                 goto out;
1484         }
1485         areq_entry = (u8 *)mrioc->admin_req_base +
1486             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1487         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1488         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1489
1490         if (++areq_pi == max_entries)
1491                 areq_pi = 0;
1492         mrioc->admin_req_pi = areq_pi;
1493
1494         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1495
1496 out:
1497         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1498
1499         return retval;
1500 }
1501
1502 /**
1503  * mpi3mr_free_op_req_q_segments - free request memory segments
1504  * @mrioc: Adapter instance reference
1505  * @q_idx: operational request queue index
1506  *
1507  * Free memory segments allocated for operational request queue
1508  *
1509  * Return: Nothing.
1510  */
1511 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1512 {
1513         u16 j;
1514         int size;
1515         struct segments *segments;
1516
1517         segments = mrioc->req_qinfo[q_idx].q_segments;
1518         if (!segments)
1519                 return;
1520
1521         if (mrioc->enable_segqueue) {
1522                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1523                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1524                         dma_free_coherent(&mrioc->pdev->dev,
1525                             MPI3MR_MAX_SEG_LIST_SIZE,
1526                             mrioc->req_qinfo[q_idx].q_segment_list,
1527                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1528                         mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1529                 }
1530         } else
1531                 size = mrioc->req_qinfo[q_idx].segment_qd *
1532                     mrioc->facts.op_req_sz;
1533
1534         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1535                 if (!segments[j].segment)
1536                         continue;
1537                 dma_free_coherent(&mrioc->pdev->dev,
1538                     size, segments[j].segment, segments[j].segment_dma);
1539                 segments[j].segment = NULL;
1540         }
1541         kfree(mrioc->req_qinfo[q_idx].q_segments);
1542         mrioc->req_qinfo[q_idx].q_segments = NULL;
1543         mrioc->req_qinfo[q_idx].qid = 0;
1544 }
1545
1546 /**
1547  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1548  * @mrioc: Adapter instance reference
1549  * @q_idx: operational reply queue index
1550  *
1551  * Free memory segments allocated for operational reply queue
1552  *
1553  * Return: Nothing.
1554  */
1555 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1556 {
1557         u16 j;
1558         int size;
1559         struct segments *segments;
1560
1561         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1562         if (!segments)
1563                 return;
1564
1565         if (mrioc->enable_segqueue) {
1566                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1567                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1568                         dma_free_coherent(&mrioc->pdev->dev,
1569                             MPI3MR_MAX_SEG_LIST_SIZE,
1570                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1571                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1572                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1573                 }
1574         } else
1575                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1576                     mrioc->op_reply_desc_sz;
1577
1578         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1579                 if (!segments[j].segment)
1580                         continue;
1581                 dma_free_coherent(&mrioc->pdev->dev,
1582                     size, segments[j].segment, segments[j].segment_dma);
1583                 segments[j].segment = NULL;
1584         }
1585
1586         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1587         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1588         mrioc->op_reply_qinfo[q_idx].qid = 0;
1589 }
1590
1591 /**
1592  * mpi3mr_delete_op_reply_q - delete operational reply queue
1593  * @mrioc: Adapter instance reference
1594  * @qidx: operational reply queue index
1595  *
1596  * Delete operatinal reply queue by issuing MPI request
1597  * through admin queue.
1598  *
1599  * Return:  0 on success, non-zero on failure.
1600  */
1601 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1602 {
1603         struct mpi3_delete_reply_queue_request delq_req;
1604         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1605         int retval = 0;
1606         u16 reply_qid = 0, midx;
1607
1608         reply_qid = op_reply_q->qid;
1609
1610         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1611
1612         if (!reply_qid) {
1613                 retval = -1;
1614                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1615                 goto out;
1616         }
1617
1618         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1619             mrioc->active_poll_qcount--;
1620
1621         memset(&delq_req, 0, sizeof(delq_req));
1622         mutex_lock(&mrioc->init_cmds.mutex);
1623         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1624                 retval = -1;
1625                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1626                 mutex_unlock(&mrioc->init_cmds.mutex);
1627                 goto out;
1628         }
1629         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1630         mrioc->init_cmds.is_waiting = 1;
1631         mrioc->init_cmds.callback = NULL;
1632         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1633         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1634         delq_req.queue_id = cpu_to_le16(reply_qid);
1635
1636         init_completion(&mrioc->init_cmds.done);
1637         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1638             1);
1639         if (retval) {
1640                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1641                 goto out_unlock;
1642         }
1643         wait_for_completion_timeout(&mrioc->init_cmds.done,
1644             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1645         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1646                 ioc_err(mrioc, "delete reply queue timed out\n");
1647                 mpi3mr_check_rh_fault_ioc(mrioc,
1648                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1649                 retval = -1;
1650                 goto out_unlock;
1651         }
1652         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1653             != MPI3_IOCSTATUS_SUCCESS) {
1654                 ioc_err(mrioc,
1655                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1656                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1657                     mrioc->init_cmds.ioc_loginfo);
1658                 retval = -1;
1659                 goto out_unlock;
1660         }
1661         mrioc->intr_info[midx].op_reply_q = NULL;
1662
1663         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1664 out_unlock:
1665         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1666         mutex_unlock(&mrioc->init_cmds.mutex);
1667 out:
1668
1669         return retval;
1670 }
1671
1672 /**
1673  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1674  * @mrioc: Adapter instance reference
1675  * @qidx: request queue index
1676  *
1677  * Allocate segmented memory pools for operational reply
1678  * queue.
1679  *
1680  * Return: 0 on success, non-zero on failure.
1681  */
1682 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1683 {
1684         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1685         int i, size;
1686         u64 *q_segment_list_entry = NULL;
1687         struct segments *segments;
1688
1689         if (mrioc->enable_segqueue) {
1690                 op_reply_q->segment_qd =
1691                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1692
1693                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1694
1695                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1696                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1697                     GFP_KERNEL);
1698                 if (!op_reply_q->q_segment_list)
1699                         return -ENOMEM;
1700                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1701         } else {
1702                 op_reply_q->segment_qd = op_reply_q->num_replies;
1703                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1704         }
1705
1706         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1707             op_reply_q->segment_qd);
1708
1709         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1710             sizeof(struct segments), GFP_KERNEL);
1711         if (!op_reply_q->q_segments)
1712                 return -ENOMEM;
1713
1714         segments = op_reply_q->q_segments;
1715         for (i = 0; i < op_reply_q->num_segments; i++) {
1716                 segments[i].segment =
1717                     dma_alloc_coherent(&mrioc->pdev->dev,
1718                     size, &segments[i].segment_dma, GFP_KERNEL);
1719                 if (!segments[i].segment)
1720                         return -ENOMEM;
1721                 if (mrioc->enable_segqueue)
1722                         q_segment_list_entry[i] =
1723                             (unsigned long)segments[i].segment_dma;
1724         }
1725
1726         return 0;
1727 }
1728
1729 /**
1730  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1731  * @mrioc: Adapter instance reference
1732  * @qidx: request queue index
1733  *
1734  * Allocate segmented memory pools for operational request
1735  * queue.
1736  *
1737  * Return: 0 on success, non-zero on failure.
1738  */
1739 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1740 {
1741         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1742         int i, size;
1743         u64 *q_segment_list_entry = NULL;
1744         struct segments *segments;
1745
1746         if (mrioc->enable_segqueue) {
1747                 op_req_q->segment_qd =
1748                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1749
1750                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1751
1752                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1753                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1754                     GFP_KERNEL);
1755                 if (!op_req_q->q_segment_list)
1756                         return -ENOMEM;
1757                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1758
1759         } else {
1760                 op_req_q->segment_qd = op_req_q->num_requests;
1761                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1762         }
1763
1764         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1765             op_req_q->segment_qd);
1766
1767         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1768             sizeof(struct segments), GFP_KERNEL);
1769         if (!op_req_q->q_segments)
1770                 return -ENOMEM;
1771
1772         segments = op_req_q->q_segments;
1773         for (i = 0; i < op_req_q->num_segments; i++) {
1774                 segments[i].segment =
1775                     dma_alloc_coherent(&mrioc->pdev->dev,
1776                     size, &segments[i].segment_dma, GFP_KERNEL);
1777                 if (!segments[i].segment)
1778                         return -ENOMEM;
1779                 if (mrioc->enable_segqueue)
1780                         q_segment_list_entry[i] =
1781                             (unsigned long)segments[i].segment_dma;
1782         }
1783
1784         return 0;
1785 }
1786
1787 /**
1788  * mpi3mr_create_op_reply_q - create operational reply queue
1789  * @mrioc: Adapter instance reference
1790  * @qidx: operational reply queue index
1791  *
1792  * Create operatinal reply queue by issuing MPI request
1793  * through admin queue.
1794  *
1795  * Return:  0 on success, non-zero on failure.
1796  */
1797 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1798 {
1799         struct mpi3_create_reply_queue_request create_req;
1800         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1801         int retval = 0;
1802         u16 reply_qid = 0, midx;
1803
1804         reply_qid = op_reply_q->qid;
1805
1806         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1807
1808         if (reply_qid) {
1809                 retval = -1;
1810                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1811                     reply_qid);
1812
1813                 return retval;
1814         }
1815
1816         reply_qid = qidx + 1;
1817         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1818         if (!mrioc->pdev->revision)
1819                 op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
1820         op_reply_q->ci = 0;
1821         op_reply_q->ephase = 1;
1822         atomic_set(&op_reply_q->pend_ios, 0);
1823         atomic_set(&op_reply_q->in_use, 0);
1824         op_reply_q->enable_irq_poll = false;
1825
1826         if (!op_reply_q->q_segments) {
1827                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1828                 if (retval) {
1829                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1830                         goto out;
1831                 }
1832         }
1833
1834         memset(&create_req, 0, sizeof(create_req));
1835         mutex_lock(&mrioc->init_cmds.mutex);
1836         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1837                 retval = -1;
1838                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1839                 goto out_unlock;
1840         }
1841         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1842         mrioc->init_cmds.is_waiting = 1;
1843         mrioc->init_cmds.callback = NULL;
1844         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1845         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1846         create_req.queue_id = cpu_to_le16(reply_qid);
1847
1848         if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
1849                 op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;
1850         else
1851                 op_reply_q->qtype = MPI3MR_POLL_QUEUE;
1852
1853         if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {
1854                 create_req.flags =
1855                         MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1856                 create_req.msix_index =
1857                         cpu_to_le16(mrioc->intr_info[midx].msix_index);
1858         } else {
1859                 create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
1860                 ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
1861                         reply_qid, midx);
1862                 if (!mrioc->active_poll_qcount)
1863                         disable_irq_nosync(pci_irq_vector(mrioc->pdev,
1864                             mrioc->intr_info_count - 1));
1865         }
1866
1867         if (mrioc->enable_segqueue) {
1868                 create_req.flags |=
1869                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1870                 create_req.base_address = cpu_to_le64(
1871                     op_reply_q->q_segment_list_dma);
1872         } else
1873                 create_req.base_address = cpu_to_le64(
1874                     op_reply_q->q_segments[0].segment_dma);
1875
1876         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1877
1878         init_completion(&mrioc->init_cmds.done);
1879         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1880             sizeof(create_req), 1);
1881         if (retval) {
1882                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1883                 goto out_unlock;
1884         }
1885         wait_for_completion_timeout(&mrioc->init_cmds.done,
1886             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1887         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1888                 ioc_err(mrioc, "create reply queue timed out\n");
1889                 mpi3mr_check_rh_fault_ioc(mrioc,
1890                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1891                 retval = -1;
1892                 goto out_unlock;
1893         }
1894         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1895             != MPI3_IOCSTATUS_SUCCESS) {
1896                 ioc_err(mrioc,
1897                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1898                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1899                     mrioc->init_cmds.ioc_loginfo);
1900                 retval = -1;
1901                 goto out_unlock;
1902         }
1903         op_reply_q->qid = reply_qid;
1904         if (midx < mrioc->intr_info_count)
1905                 mrioc->intr_info[midx].op_reply_q = op_reply_q;
1906
1907         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
1908             mrioc->active_poll_qcount++;
1909
1910 out_unlock:
1911         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1912         mutex_unlock(&mrioc->init_cmds.mutex);
1913 out:
1914
1915         return retval;
1916 }
1917
1918 /**
1919  * mpi3mr_create_op_req_q - create operational request queue
1920  * @mrioc: Adapter instance reference
1921  * @idx: operational request queue index
1922  * @reply_qid: Reply queue ID
1923  *
1924  * Create operatinal request queue by issuing MPI request
1925  * through admin queue.
1926  *
1927  * Return:  0 on success, non-zero on failure.
1928  */
1929 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1930         u16 reply_qid)
1931 {
1932         struct mpi3_create_request_queue_request create_req;
1933         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1934         int retval = 0;
1935         u16 req_qid = 0;
1936
1937         req_qid = op_req_q->qid;
1938
1939         if (req_qid) {
1940                 retval = -1;
1941                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1942                     req_qid);
1943
1944                 return retval;
1945         }
1946         req_qid = idx + 1;
1947
1948         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1949         op_req_q->ci = 0;
1950         op_req_q->pi = 0;
1951         op_req_q->reply_qid = reply_qid;
1952         spin_lock_init(&op_req_q->q_lock);
1953
1954         if (!op_req_q->q_segments) {
1955                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1956                 if (retval) {
1957                         mpi3mr_free_op_req_q_segments(mrioc, idx);
1958                         goto out;
1959                 }
1960         }
1961
1962         memset(&create_req, 0, sizeof(create_req));
1963         mutex_lock(&mrioc->init_cmds.mutex);
1964         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1965                 retval = -1;
1966                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
1967                 goto out_unlock;
1968         }
1969         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1970         mrioc->init_cmds.is_waiting = 1;
1971         mrioc->init_cmds.callback = NULL;
1972         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1973         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
1974         create_req.queue_id = cpu_to_le16(req_qid);
1975         if (mrioc->enable_segqueue) {
1976                 create_req.flags =
1977                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1978                 create_req.base_address = cpu_to_le64(
1979                     op_req_q->q_segment_list_dma);
1980         } else
1981                 create_req.base_address = cpu_to_le64(
1982                     op_req_q->q_segments[0].segment_dma);
1983         create_req.reply_queue_id = cpu_to_le16(reply_qid);
1984         create_req.size = cpu_to_le16(op_req_q->num_requests);
1985
1986         init_completion(&mrioc->init_cmds.done);
1987         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1988             sizeof(create_req), 1);
1989         if (retval) {
1990                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
1991                 goto out_unlock;
1992         }
1993         wait_for_completion_timeout(&mrioc->init_cmds.done,
1994             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1995         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1996                 ioc_err(mrioc, "create request queue timed out\n");
1997                 mpi3mr_check_rh_fault_ioc(mrioc,
1998                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
1999                 retval = -1;
2000                 goto out_unlock;
2001         }
2002         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2003             != MPI3_IOCSTATUS_SUCCESS) {
2004                 ioc_err(mrioc,
2005                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2006                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2007                     mrioc->init_cmds.ioc_loginfo);
2008                 retval = -1;
2009                 goto out_unlock;
2010         }
2011         op_req_q->qid = req_qid;
2012
2013 out_unlock:
2014         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2015         mutex_unlock(&mrioc->init_cmds.mutex);
2016 out:
2017
2018         return retval;
2019 }
2020
2021 /**
2022  * mpi3mr_create_op_queues - create operational queue pairs
2023  * @mrioc: Adapter instance reference
2024  *
2025  * Allocate memory for operational queue meta data and call
2026  * create request and reply queue functions.
2027  *
2028  * Return: 0 on success, non-zero on failures.
2029  */
2030 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2031 {
2032         int retval = 0;
2033         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
2034
2035         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2036             mrioc->facts.max_op_req_q);
2037
2038         msix_count_op_q =
2039             mrioc->intr_info_count - mrioc->op_reply_q_offset;
2040         if (!mrioc->num_queues)
2041                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2042         /*
2043          * During reset set the num_queues to the number of queues
2044          * that was set before the reset.
2045          */
2046         num_queues = mrioc->num_op_reply_q ?
2047             mrioc->num_op_reply_q : mrioc->num_queues;
2048         ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2049             num_queues);
2050
2051         if (!mrioc->req_qinfo) {
2052                 mrioc->req_qinfo = kcalloc(num_queues,
2053                     sizeof(struct op_req_qinfo), GFP_KERNEL);
2054                 if (!mrioc->req_qinfo) {
2055                         retval = -1;
2056                         goto out_failed;
2057                 }
2058
2059                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
2060                     num_queues, GFP_KERNEL);
2061                 if (!mrioc->op_reply_qinfo) {
2062                         retval = -1;
2063                         goto out_failed;
2064                 }
2065         }
2066
2067         if (mrioc->enable_segqueue)
2068                 ioc_info(mrioc,
2069                     "allocating operational queues through segmented queues\n");
2070
2071         for (i = 0; i < num_queues; i++) {
2072                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
2073                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2074                         break;
2075                 }
2076                 if (mpi3mr_create_op_req_q(mrioc, i,
2077                     mrioc->op_reply_qinfo[i].qid)) {
2078                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2079                         mpi3mr_delete_op_reply_q(mrioc, i);
2080                         break;
2081                 }
2082         }
2083
2084         if (i == 0) {
2085                 /* Not even one queue is created successfully*/
2086                 retval = -1;
2087                 goto out_failed;
2088         }
2089         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2090         ioc_info(mrioc,
2091             "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",
2092             mrioc->num_op_reply_q, mrioc->default_qcount,
2093             mrioc->active_poll_qcount);
2094
2095         return retval;
2096 out_failed:
2097         kfree(mrioc->req_qinfo);
2098         mrioc->req_qinfo = NULL;
2099
2100         kfree(mrioc->op_reply_qinfo);
2101         mrioc->op_reply_qinfo = NULL;
2102
2103         return retval;
2104 }
2105
2106 /**
2107  * mpi3mr_op_request_post - Post request to operational queue
2108  * @mrioc: Adapter reference
2109  * @op_req_q: Operational request queue info
2110  * @req: MPI3 request
2111  *
2112  * Post the MPI3 request into operational request queue and
2113  * inform the controller, if the queue is full return
2114  * appropriate error.
2115  *
2116  * Return: 0 on success, non-zero on failure.
2117  */
2118 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2119         struct op_req_qinfo *op_req_q, u8 *req)
2120 {
2121         u16 pi = 0, max_entries, reply_qidx = 0, midx;
2122         int retval = 0;
2123         unsigned long flags;
2124         u8 *req_entry;
2125         void *segment_base_addr;
2126         u16 req_sz = mrioc->facts.op_req_sz;
2127         struct segments *segments = op_req_q->q_segments;
2128
2129         reply_qidx = op_req_q->reply_qid - 1;
2130
2131         if (mrioc->unrecoverable)
2132                 return -EFAULT;
2133
2134         spin_lock_irqsave(&op_req_q->q_lock, flags);
2135         pi = op_req_q->pi;
2136         max_entries = op_req_q->num_requests;
2137
2138         if (mpi3mr_check_req_qfull(op_req_q)) {
2139                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
2140                     reply_qidx, mrioc->op_reply_q_offset);
2141                 mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
2142
2143                 if (mpi3mr_check_req_qfull(op_req_q)) {
2144                         retval = -EAGAIN;
2145                         goto out;
2146                 }
2147         }
2148
2149         if (mrioc->reset_in_progress) {
2150                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2151                 retval = -EAGAIN;
2152                 goto out;
2153         }
2154
2155         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
2156         req_entry = (u8 *)segment_base_addr +
2157             ((pi % op_req_q->segment_qd) * req_sz);
2158
2159         memset(req_entry, 0, req_sz);
2160         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
2161
2162         if (++pi == max_entries)
2163                 pi = 0;
2164         op_req_q->pi = pi;
2165
2166         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2167             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
2168                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2169
2170         writel(op_req_q->pi,
2171             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2172
2173 out:
2174         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
2175         return retval;
2176 }
2177
2178 /**
2179  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2180  * controller
2181  * @mrioc: Adapter instance reference
2182  * @reason_code: reason code for the fault.
2183  *
2184  * This routine will save snapdump and fault the controller with
2185  * the given reason code if it is not already in the fault or
2186  * not asynchronosuly reset. This will be used to handle
2187  * initilaization time faults/resets/timeout as in those cases
2188  * immediate soft reset invocation is not required.
2189  *
2190  * Return:  None.
2191  */
2192 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2193 {
2194         u32 ioc_status, host_diagnostic, timeout;
2195
2196         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2197         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
2198             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
2199                 mpi3mr_print_fault_info(mrioc);
2200                 return;
2201         }
2202         mpi3mr_set_diagsave(mrioc);
2203         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2204             reason_code);
2205         timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
2206         do {
2207                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2208                 if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
2209                         break;
2210                 msleep(100);
2211         } while (--timeout);
2212 }
2213
2214 /**
2215  * mpi3mr_sync_timestamp - Issue time stamp sync request
2216  * @mrioc: Adapter reference
2217  *
2218  * Issue IO unit control MPI request to synchornize firmware
2219  * timestamp with host time.
2220  *
2221  * Return: 0 on success, non-zero on failure.
2222  */
2223 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2224 {
2225         ktime_t current_time;
2226         struct mpi3_iounit_control_request iou_ctrl;
2227         int retval = 0;
2228
2229         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2230         mutex_lock(&mrioc->init_cmds.mutex);
2231         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2232                 retval = -1;
2233                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2234                 mutex_unlock(&mrioc->init_cmds.mutex);
2235                 goto out;
2236         }
2237         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2238         mrioc->init_cmds.is_waiting = 1;
2239         mrioc->init_cmds.callback = NULL;
2240         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2241         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2242         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
2243         current_time = ktime_get_real();
2244         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
2245
2246         init_completion(&mrioc->init_cmds.done);
2247         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
2248             sizeof(iou_ctrl), 0);
2249         if (retval) {
2250                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2251                 goto out_unlock;
2252         }
2253
2254         wait_for_completion_timeout(&mrioc->init_cmds.done,
2255             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2256         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2257                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2258                 mrioc->init_cmds.is_waiting = 0;
2259                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2260                         mpi3mr_soft_reset_handler(mrioc,
2261                             MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
2262                 retval = -1;
2263                 goto out_unlock;
2264         }
2265         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2266             != MPI3_IOCSTATUS_SUCCESS) {
2267                 ioc_err(mrioc,
2268                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2269                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2270                     mrioc->init_cmds.ioc_loginfo);
2271                 retval = -1;
2272                 goto out_unlock;
2273         }
2274
2275 out_unlock:
2276         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2277         mutex_unlock(&mrioc->init_cmds.mutex);
2278
2279 out:
2280         return retval;
2281 }
2282
2283 /**
2284  * mpi3mr_print_pkg_ver - display controller fw package version
2285  * @mrioc: Adapter reference
2286  *
2287  * Retrieve firmware package version from the component image
2288  * header of the controller flash and display it.
2289  *
2290  * Return: 0 on success and non-zero on failure.
2291  */
2292 static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2293 {
2294         struct mpi3_ci_upload_request ci_upload;
2295         int retval = -1;
2296         void *data = NULL;
2297         dma_addr_t data_dma;
2298         struct mpi3_ci_manifest_mpi *manifest;
2299         u32 data_len = sizeof(struct mpi3_ci_manifest_mpi);
2300         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2301
2302         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2303             GFP_KERNEL);
2304         if (!data)
2305                 return -ENOMEM;
2306
2307         memset(&ci_upload, 0, sizeof(ci_upload));
2308         mutex_lock(&mrioc->init_cmds.mutex);
2309         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2310                 ioc_err(mrioc, "sending get package version failed due to command in use\n");
2311                 mutex_unlock(&mrioc->init_cmds.mutex);
2312                 goto out;
2313         }
2314         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2315         mrioc->init_cmds.is_waiting = 1;
2316         mrioc->init_cmds.callback = NULL;
2317         ci_upload.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2318         ci_upload.function = MPI3_FUNCTION_CI_UPLOAD;
2319         ci_upload.msg_flags = MPI3_CI_UPLOAD_MSGFLAGS_LOCATION_PRIMARY;
2320         ci_upload.signature1 = cpu_to_le32(MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST);
2321         ci_upload.image_offset = cpu_to_le32(MPI3_IMAGE_HEADER_SIZE);
2322         ci_upload.segment_size = cpu_to_le32(data_len);
2323
2324         mpi3mr_add_sg_single(&ci_upload.sgl, sgl_flags, data_len,
2325             data_dma);
2326         init_completion(&mrioc->init_cmds.done);
2327         retval = mpi3mr_admin_request_post(mrioc, &ci_upload,
2328             sizeof(ci_upload), 1);
2329         if (retval) {
2330                 ioc_err(mrioc, "posting get package version failed\n");
2331                 goto out_unlock;
2332         }
2333         wait_for_completion_timeout(&mrioc->init_cmds.done,
2334             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2335         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2336                 ioc_err(mrioc, "get package version timed out\n");
2337                 mpi3mr_check_rh_fault_ioc(mrioc,
2338                     MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
2339                 retval = -1;
2340                 goto out_unlock;
2341         }
2342         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2343             == MPI3_IOCSTATUS_SUCCESS) {
2344                 manifest = (struct mpi3_ci_manifest_mpi *) data;
2345                 if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {
2346                         ioc_info(mrioc,
2347                             "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",
2348                             manifest->package_version.gen_major,
2349                             manifest->package_version.gen_minor,
2350                             manifest->package_version.phase_major,
2351                             manifest->package_version.phase_minor,
2352                             manifest->package_version.customer_id,
2353                             manifest->package_version.build_num);
2354                 }
2355         }
2356         retval = 0;
2357 out_unlock:
2358         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2359         mutex_unlock(&mrioc->init_cmds.mutex);
2360
2361 out:
2362         if (data)
2363                 dma_free_coherent(&mrioc->pdev->dev, data_len, data,
2364                     data_dma);
2365         return retval;
2366 }
2367
2368 /**
2369  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2370  * @work: work struct
2371  *
2372  * Watch dog work periodically executed (1 second interval) to
2373  * monitor firmware fault and to issue periodic timer sync to
2374  * the firmware.
2375  *
2376  * Return: Nothing.
2377  */
2378 static void mpi3mr_watchdog_work(struct work_struct *work)
2379 {
2380         struct mpi3mr_ioc *mrioc =
2381             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
2382         unsigned long flags;
2383         enum mpi3mr_iocstate ioc_state;
2384         u32 fault, host_diagnostic, ioc_status;
2385         u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH;
2386
2387         if (mrioc->reset_in_progress || mrioc->unrecoverable)
2388                 return;
2389
2390         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2391                 mrioc->ts_update_counter = 0;
2392                 mpi3mr_sync_timestamp(mrioc);
2393         }
2394
2395         if ((mrioc->prepare_for_reset) &&
2396             ((mrioc->prepare_for_reset_timeout_counter++) >=
2397              MPI3MR_PREPARE_FOR_RESET_TIMEOUT)) {
2398                 mpi3mr_soft_reset_handler(mrioc,
2399                     MPI3MR_RESET_FROM_CIACTVRST_TIMER, 1);
2400                 return;
2401         }
2402
2403         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2404         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
2405                 mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_FIRMWARE, 0);
2406                 return;
2407         }
2408
2409         /*Check for fault state every one second and issue Soft reset*/
2410         ioc_state = mpi3mr_get_iocstate(mrioc);
2411         if (ioc_state != MRIOC_STATE_FAULT)
2412                 goto schedule_work;
2413
2414         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2415         host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2416         if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2417                 if (!mrioc->diagsave_timeout) {
2418                         mpi3mr_print_fault_info(mrioc);
2419                         ioc_warn(mrioc, "diag save in progress\n");
2420                 }
2421                 if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2422                         goto schedule_work;
2423         }
2424
2425         mpi3mr_print_fault_info(mrioc);
2426         mrioc->diagsave_timeout = 0;
2427
2428         switch (fault) {
2429         case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED:
2430                 ioc_info(mrioc,
2431                     "controller requires system power cycle, marking controller as unrecoverable\n");
2432                 mrioc->unrecoverable = 1;
2433                 return;
2434         case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS:
2435                 return;
2436         case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET:
2437                 reset_reason = MPI3MR_RESET_FROM_CIACTIV_FAULT;
2438                 break;
2439         default:
2440                 break;
2441         }
2442         mpi3mr_soft_reset_handler(mrioc, reset_reason, 0);
2443         return;
2444
2445 schedule_work:
2446         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2447         if (mrioc->watchdog_work_q)
2448                 queue_delayed_work(mrioc->watchdog_work_q,
2449                     &mrioc->watchdog_work,
2450                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2451         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2452         return;
2453 }
2454
2455 /**
2456  * mpi3mr_start_watchdog - Start watchdog
2457  * @mrioc: Adapter instance reference
2458  *
2459  * Create and start the watchdog thread to monitor controller
2460  * faults.
2461  *
2462  * Return: Nothing.
2463  */
2464 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2465 {
2466         if (mrioc->watchdog_work_q)
2467                 return;
2468
2469         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2470         snprintf(mrioc->watchdog_work_q_name,
2471             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2472             mrioc->id);
2473         mrioc->watchdog_work_q =
2474             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2475         if (!mrioc->watchdog_work_q) {
2476                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2477                 return;
2478         }
2479
2480         if (mrioc->watchdog_work_q)
2481                 queue_delayed_work(mrioc->watchdog_work_q,
2482                     &mrioc->watchdog_work,
2483                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2484 }
2485
2486 /**
2487  * mpi3mr_stop_watchdog - Stop watchdog
2488  * @mrioc: Adapter instance reference
2489  *
2490  * Stop the watchdog thread created to monitor controller
2491  * faults.
2492  *
2493  * Return: Nothing.
2494  */
2495 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2496 {
2497         unsigned long flags;
2498         struct workqueue_struct *wq;
2499
2500         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2501         wq = mrioc->watchdog_work_q;
2502         mrioc->watchdog_work_q = NULL;
2503         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2504         if (wq) {
2505                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2506                         flush_workqueue(wq);
2507                 destroy_workqueue(wq);
2508         }
2509 }
2510
2511 /**
2512  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2513  * @mrioc: Adapter instance reference
2514  *
2515  * Allocate memory for admin queue pair if required and register
2516  * the admin queue with the controller.
2517  *
2518  * Return: 0 on success, non-zero on failures.
2519  */
2520 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2521 {
2522         int retval = 0;
2523         u32 num_admin_entries = 0;
2524
2525         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2526         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2527             MPI3MR_ADMIN_REQ_FRAME_SZ;
2528         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2529         mrioc->admin_req_base = NULL;
2530
2531         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2532         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2533             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2534         mrioc->admin_reply_ci = 0;
2535         mrioc->admin_reply_ephase = 1;
2536         mrioc->admin_reply_base = NULL;
2537
2538         if (!mrioc->admin_req_base) {
2539                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2540                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2541
2542                 if (!mrioc->admin_req_base) {
2543                         retval = -1;
2544                         goto out_failed;
2545                 }
2546
2547                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2548                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2549                     GFP_KERNEL);
2550
2551                 if (!mrioc->admin_reply_base) {
2552                         retval = -1;
2553                         goto out_failed;
2554                 }
2555         }
2556
2557         num_admin_entries = (mrioc->num_admin_replies << 16) |
2558             (mrioc->num_admin_req);
2559         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2560         mpi3mr_writeq(mrioc->admin_req_dma,
2561             &mrioc->sysif_regs->admin_request_queue_address);
2562         mpi3mr_writeq(mrioc->admin_reply_dma,
2563             &mrioc->sysif_regs->admin_reply_queue_address);
2564         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2565         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2566         return retval;
2567
2568 out_failed:
2569
2570         if (mrioc->admin_reply_base) {
2571                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2572                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2573                 mrioc->admin_reply_base = NULL;
2574         }
2575         if (mrioc->admin_req_base) {
2576                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2577                     mrioc->admin_req_base, mrioc->admin_req_dma);
2578                 mrioc->admin_req_base = NULL;
2579         }
2580         return retval;
2581 }
2582
2583 /**
2584  * mpi3mr_issue_iocfacts - Send IOC Facts
2585  * @mrioc: Adapter instance reference
2586  * @facts_data: Cached IOC facts data
2587  *
2588  * Issue IOC Facts MPI request through admin queue and wait for
2589  * the completion of it or time out.
2590  *
2591  * Return: 0 on success, non-zero on failures.
2592  */
2593 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2594         struct mpi3_ioc_facts_data *facts_data)
2595 {
2596         struct mpi3_ioc_facts_request iocfacts_req;
2597         void *data = NULL;
2598         dma_addr_t data_dma;
2599         u32 data_len = sizeof(*facts_data);
2600         int retval = 0;
2601         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2602
2603         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2604             GFP_KERNEL);
2605
2606         if (!data) {
2607                 retval = -1;
2608                 goto out;
2609         }
2610
2611         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2612         mutex_lock(&mrioc->init_cmds.mutex);
2613         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2614                 retval = -1;
2615                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2616                 mutex_unlock(&mrioc->init_cmds.mutex);
2617                 goto out;
2618         }
2619         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2620         mrioc->init_cmds.is_waiting = 1;
2621         mrioc->init_cmds.callback = NULL;
2622         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2623         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2624
2625         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2626             data_dma);
2627
2628         init_completion(&mrioc->init_cmds.done);
2629         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2630             sizeof(iocfacts_req), 1);
2631         if (retval) {
2632                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2633                 goto out_unlock;
2634         }
2635         wait_for_completion_timeout(&mrioc->init_cmds.done,
2636             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2637         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2638                 ioc_err(mrioc, "ioc_facts timed out\n");
2639                 mpi3mr_check_rh_fault_ioc(mrioc,
2640                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2641                 retval = -1;
2642                 goto out_unlock;
2643         }
2644         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2645             != MPI3_IOCSTATUS_SUCCESS) {
2646                 ioc_err(mrioc,
2647                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2648                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2649                     mrioc->init_cmds.ioc_loginfo);
2650                 retval = -1;
2651                 goto out_unlock;
2652         }
2653         memcpy(facts_data, (u8 *)data, data_len);
2654         mpi3mr_process_factsdata(mrioc, facts_data);
2655 out_unlock:
2656         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2657         mutex_unlock(&mrioc->init_cmds.mutex);
2658
2659 out:
2660         if (data)
2661                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2662
2663         return retval;
2664 }
2665
2666 /**
2667  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2668  * @mrioc: Adapter instance reference
2669  *
2670  * Check whether the new DMA mask requested through IOCFacts by
2671  * firmware needs to be set, if so set it .
2672  *
2673  * Return: 0 on success, non-zero on failure.
2674  */
2675 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2676 {
2677         struct pci_dev *pdev = mrioc->pdev;
2678         int r;
2679         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2680
2681         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2682                 return 0;
2683
2684         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2685             mrioc->dma_mask, facts_dma_mask);
2686
2687         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2688         if (r) {
2689                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2690                     facts_dma_mask, r);
2691                 return r;
2692         }
2693         mrioc->dma_mask = facts_dma_mask;
2694         return r;
2695 }
2696
2697 /**
2698  * mpi3mr_process_factsdata - Process IOC facts data
2699  * @mrioc: Adapter instance reference
2700  * @facts_data: Cached IOC facts data
2701  *
2702  * Convert IOC facts data into cpu endianness and cache it in
2703  * the driver .
2704  *
2705  * Return: Nothing.
2706  */
2707 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2708         struct mpi3_ioc_facts_data *facts_data)
2709 {
2710         u32 ioc_config, req_sz, facts_flags;
2711
2712         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2713             (sizeof(*facts_data) / 4)) {
2714                 ioc_warn(mrioc,
2715                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2716                     sizeof(*facts_data),
2717                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2718         }
2719
2720         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2721         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2722             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2723         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2724                 ioc_err(mrioc,
2725                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2726                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2727         }
2728
2729         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2730
2731         facts_flags = le32_to_cpu(facts_data->flags);
2732         mrioc->facts.op_req_sz = req_sz;
2733         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2734             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2735             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2736
2737         mrioc->facts.ioc_num = facts_data->ioc_number;
2738         mrioc->facts.who_init = facts_data->who_init;
2739         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2740         mrioc->facts.personality = (facts_flags &
2741             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2742         mrioc->facts.dma_mask = (facts_flags &
2743             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2744             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2745         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2746         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2747         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2748         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2749         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2750         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2751         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2752         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2753         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2754         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2755         mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2756         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2757         mrioc->facts.max_pcie_switches =
2758             le16_to_cpu(facts_data->max_pcie_switches);
2759         mrioc->facts.max_sasexpanders =
2760             le16_to_cpu(facts_data->max_sas_expanders);
2761         mrioc->facts.max_sasinitiators =
2762             le16_to_cpu(facts_data->max_sas_initiators);
2763         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2764         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2765         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2766         mrioc->facts.max_op_req_q =
2767             le16_to_cpu(facts_data->max_operational_request_queues);
2768         mrioc->facts.max_op_reply_q =
2769             le16_to_cpu(facts_data->max_operational_reply_queues);
2770         mrioc->facts.ioc_capabilities =
2771             le32_to_cpu(facts_data->ioc_capabilities);
2772         mrioc->facts.fw_ver.build_num =
2773             le16_to_cpu(facts_data->fw_version.build_num);
2774         mrioc->facts.fw_ver.cust_id =
2775             le16_to_cpu(facts_data->fw_version.customer_id);
2776         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2777         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2778         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2779         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2780         mrioc->msix_count = min_t(int, mrioc->msix_count,
2781             mrioc->facts.max_msix_vectors);
2782         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2783         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2784         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2785         mrioc->facts.shutdown_timeout =
2786             le16_to_cpu(facts_data->shutdown_timeout);
2787
2788         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2789             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2790             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2791         ioc_info(mrioc,
2792             "maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
2793             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2794             mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
2795         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2796             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2797             mrioc->facts.sge_mod_shift);
2798         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2799             mrioc->facts.dma_mask, (facts_flags &
2800             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2801 }
2802
2803 /**
2804  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2805  * @mrioc: Adapter instance reference
2806  *
2807  * Allocate and initialize the reply free buffers, sense
2808  * buffers, reply free queue and sense buffer queue.
2809  *
2810  * Return: 0 on success, non-zero on failures.
2811  */
2812 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2813 {
2814         int retval = 0;
2815         u32 sz, i;
2816
2817         if (mrioc->init_cmds.reply)
2818                 return retval;
2819
2820         mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2821         if (!mrioc->init_cmds.reply)
2822                 goto out_failed;
2823
2824         mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2825         if (!mrioc->bsg_cmds.reply)
2826                 goto out_failed;
2827
2828         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2829                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,
2830                     GFP_KERNEL);
2831                 if (!mrioc->dev_rmhs_cmds[i].reply)
2832                         goto out_failed;
2833         }
2834
2835         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
2836                 mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,
2837                     GFP_KERNEL);
2838                 if (!mrioc->evtack_cmds[i].reply)
2839                         goto out_failed;
2840         }
2841
2842         mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2843         if (!mrioc->host_tm_cmds.reply)
2844                 goto out_failed;
2845
2846         mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2847         if (!mrioc->pel_cmds.reply)
2848                 goto out_failed;
2849
2850         mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2851         if (!mrioc->pel_abort_cmd.reply)
2852                 goto out_failed;
2853
2854         mrioc->dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
2855         if (mrioc->facts.max_devhandle % 8)
2856                 mrioc->dev_handle_bitmap_sz++;
2857         mrioc->removepend_bitmap = kzalloc(mrioc->dev_handle_bitmap_sz,
2858             GFP_KERNEL);
2859         if (!mrioc->removepend_bitmap)
2860                 goto out_failed;
2861
2862         mrioc->devrem_bitmap_sz = MPI3MR_NUM_DEVRMCMD / 8;
2863         if (MPI3MR_NUM_DEVRMCMD % 8)
2864                 mrioc->devrem_bitmap_sz++;
2865         mrioc->devrem_bitmap = kzalloc(mrioc->devrem_bitmap_sz,
2866             GFP_KERNEL);
2867         if (!mrioc->devrem_bitmap)
2868                 goto out_failed;
2869
2870         mrioc->evtack_cmds_bitmap_sz = MPI3MR_NUM_EVTACKCMD / 8;
2871         if (MPI3MR_NUM_EVTACKCMD % 8)
2872                 mrioc->evtack_cmds_bitmap_sz++;
2873         mrioc->evtack_cmds_bitmap = kzalloc(mrioc->evtack_cmds_bitmap_sz,
2874             GFP_KERNEL);
2875         if (!mrioc->evtack_cmds_bitmap)
2876                 goto out_failed;
2877
2878         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2879         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2880         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2881         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2882
2883         /* reply buffer pool, 16 byte align */
2884         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
2885         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2886             &mrioc->pdev->dev, sz, 16, 0);
2887         if (!mrioc->reply_buf_pool) {
2888                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2889                 goto out_failed;
2890         }
2891
2892         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2893             &mrioc->reply_buf_dma);
2894         if (!mrioc->reply_buf)
2895                 goto out_failed;
2896
2897         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2898
2899         /* reply free queue, 8 byte align */
2900         sz = mrioc->reply_free_qsz * 8;
2901         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2902             &mrioc->pdev->dev, sz, 8, 0);
2903         if (!mrioc->reply_free_q_pool) {
2904                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2905                 goto out_failed;
2906         }
2907         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
2908             GFP_KERNEL, &mrioc->reply_free_q_dma);
2909         if (!mrioc->reply_free_q)
2910                 goto out_failed;
2911
2912         /* sense buffer pool,  4 byte align */
2913         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
2914         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
2915             &mrioc->pdev->dev, sz, 4, 0);
2916         if (!mrioc->sense_buf_pool) {
2917                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
2918                 goto out_failed;
2919         }
2920         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
2921             &mrioc->sense_buf_dma);
2922         if (!mrioc->sense_buf)
2923                 goto out_failed;
2924
2925         /* sense buffer queue, 8 byte align */
2926         sz = mrioc->sense_buf_q_sz * 8;
2927         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
2928             &mrioc->pdev->dev, sz, 8, 0);
2929         if (!mrioc->sense_buf_q_pool) {
2930                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
2931                 goto out_failed;
2932         }
2933         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
2934             GFP_KERNEL, &mrioc->sense_buf_q_dma);
2935         if (!mrioc->sense_buf_q)
2936                 goto out_failed;
2937
2938         return retval;
2939
2940 out_failed:
2941         retval = -1;
2942         return retval;
2943 }
2944
2945 /**
2946  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
2947  * buffers
2948  * @mrioc: Adapter instance reference
2949  *
2950  * Helper function to initialize reply and sense buffers along
2951  * with some debug prints.
2952  *
2953  * Return:  None.
2954  */
2955 static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
2956 {
2957         u32 sz, i;
2958         dma_addr_t phy_addr;
2959
2960         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
2961         ioc_info(mrioc,
2962             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2963             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
2964             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
2965         sz = mrioc->reply_free_qsz * 8;
2966         ioc_info(mrioc,
2967             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2968             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
2969             (unsigned long long)mrioc->reply_free_q_dma);
2970         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
2971         ioc_info(mrioc,
2972             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2973             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
2974             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
2975         sz = mrioc->sense_buf_q_sz * 8;
2976         ioc_info(mrioc,
2977             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2978             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
2979             (unsigned long long)mrioc->sense_buf_q_dma);
2980
2981         /* initialize Reply buffer Queue */
2982         for (i = 0, phy_addr = mrioc->reply_buf_dma;
2983             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
2984                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
2985         mrioc->reply_free_q[i] = cpu_to_le64(0);
2986
2987         /* initialize Sense Buffer Queue */
2988         for (i = 0, phy_addr = mrioc->sense_buf_dma;
2989             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
2990                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
2991         mrioc->sense_buf_q[i] = cpu_to_le64(0);
2992 }
2993
2994 /**
2995  * mpi3mr_issue_iocinit - Send IOC Init
2996  * @mrioc: Adapter instance reference
2997  *
2998  * Issue IOC Init MPI request through admin queue and wait for
2999  * the completion of it or time out.
3000  *
3001  * Return: 0 on success, non-zero on failures.
3002  */
3003 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3004 {
3005         struct mpi3_ioc_init_request iocinit_req;
3006         struct mpi3_driver_info_layout *drv_info;
3007         dma_addr_t data_dma;
3008         u32 data_len = sizeof(*drv_info);
3009         int retval = 0;
3010         ktime_t current_time;
3011
3012         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
3013             GFP_KERNEL);
3014         if (!drv_info) {
3015                 retval = -1;
3016                 goto out;
3017         }
3018         mpimr_initialize_reply_sbuf_queues(mrioc);
3019
3020         drv_info->information_length = cpu_to_le32(data_len);
3021         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
3022         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
3023         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
3024         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
3025         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
3026         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
3027             sizeof(drv_info->driver_release_date));
3028         drv_info->driver_capabilities = 0;
3029         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3030             sizeof(mrioc->driver_info));
3031
3032         memset(&iocinit_req, 0, sizeof(iocinit_req));
3033         mutex_lock(&mrioc->init_cmds.mutex);
3034         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3035                 retval = -1;
3036                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3037                 mutex_unlock(&mrioc->init_cmds.mutex);
3038                 goto out;
3039         }
3040         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3041         mrioc->init_cmds.is_waiting = 1;
3042         mrioc->init_cmds.callback = NULL;
3043         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3044         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
3045         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
3046         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
3047         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
3048         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
3049         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
3050         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3051         iocinit_req.reply_free_queue_address =
3052             cpu_to_le64(mrioc->reply_free_q_dma);
3053         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
3054         iocinit_req.sense_buffer_free_queue_depth =
3055             cpu_to_le16(mrioc->sense_buf_q_sz);
3056         iocinit_req.sense_buffer_free_queue_address =
3057             cpu_to_le64(mrioc->sense_buf_q_dma);
3058         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
3059
3060         current_time = ktime_get_real();
3061         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
3062
3063         init_completion(&mrioc->init_cmds.done);
3064         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
3065             sizeof(iocinit_req), 1);
3066         if (retval) {
3067                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3068                 goto out_unlock;
3069         }
3070         wait_for_completion_timeout(&mrioc->init_cmds.done,
3071             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3072         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3073                 mpi3mr_check_rh_fault_ioc(mrioc,
3074                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
3075                 ioc_err(mrioc, "ioc_init timed out\n");
3076                 retval = -1;
3077                 goto out_unlock;
3078         }
3079         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3080             != MPI3_IOCSTATUS_SUCCESS) {
3081                 ioc_err(mrioc,
3082                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3083                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3084                     mrioc->init_cmds.ioc_loginfo);
3085                 retval = -1;
3086                 goto out_unlock;
3087         }
3088
3089         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3090         writel(mrioc->reply_free_queue_host_index,
3091             &mrioc->sysif_regs->reply_free_host_index);
3092
3093         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3094         writel(mrioc->sbq_host_index,
3095             &mrioc->sysif_regs->sense_buffer_free_host_index);
3096 out_unlock:
3097         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3098         mutex_unlock(&mrioc->init_cmds.mutex);
3099
3100 out:
3101         if (drv_info)
3102                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
3103                     data_dma);
3104
3105         return retval;
3106 }
3107
3108 /**
3109  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3110  * @mrioc: Adapter instance reference
3111  * @event: MPI event ID
3112  *
3113  * Un mask the specific event by resetting the event_mask
3114  * bitmap.
3115  *
3116  * Return: 0 on success, non-zero on failures.
3117  */
3118 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3119 {
3120         u32 desired_event;
3121         u8 word;
3122
3123         if (event >= 128)
3124                 return;
3125
3126         desired_event = (1 << (event % 32));
3127         word = event / 32;
3128
3129         mrioc->event_masks[word] &= ~desired_event;
3130 }
3131
3132 /**
3133  * mpi3mr_issue_event_notification - Send event notification
3134  * @mrioc: Adapter instance reference
3135  *
3136  * Issue event notification MPI request through admin queue and
3137  * wait for the completion of it or time out.
3138  *
3139  * Return: 0 on success, non-zero on failures.
3140  */
3141 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3142 {
3143         struct mpi3_event_notification_request evtnotify_req;
3144         int retval = 0;
3145         u8 i;
3146
3147         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
3148         mutex_lock(&mrioc->init_cmds.mutex);
3149         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3150                 retval = -1;
3151                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3152                 mutex_unlock(&mrioc->init_cmds.mutex);
3153                 goto out;
3154         }
3155         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3156         mrioc->init_cmds.is_waiting = 1;
3157         mrioc->init_cmds.callback = NULL;
3158         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3159         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
3160         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3161                 evtnotify_req.event_masks[i] =
3162                     cpu_to_le32(mrioc->event_masks[i]);
3163         init_completion(&mrioc->init_cmds.done);
3164         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
3165             sizeof(evtnotify_req), 1);
3166         if (retval) {
3167                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3168                 goto out_unlock;
3169         }
3170         wait_for_completion_timeout(&mrioc->init_cmds.done,
3171             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3172         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3173                 ioc_err(mrioc, "event notification timed out\n");
3174                 mpi3mr_check_rh_fault_ioc(mrioc,
3175                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
3176                 retval = -1;
3177                 goto out_unlock;
3178         }
3179         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3180             != MPI3_IOCSTATUS_SUCCESS) {
3181                 ioc_err(mrioc,
3182                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3183                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3184                     mrioc->init_cmds.ioc_loginfo);
3185                 retval = -1;
3186                 goto out_unlock;
3187         }
3188
3189 out_unlock:
3190         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3191         mutex_unlock(&mrioc->init_cmds.mutex);
3192 out:
3193         return retval;
3194 }
3195
3196 /**
3197  * mpi3mr_process_event_ack - Process event acknowledgment
3198  * @mrioc: Adapter instance reference
3199  * @event: MPI3 event ID
3200  * @event_ctx: event context
3201  *
3202  * Send event acknowledgment through admin queue and wait for
3203  * it to complete.
3204  *
3205  * Return: 0 on success, non-zero on failures.
3206  */
3207 int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3208         u32 event_ctx)
3209 {
3210         struct mpi3_event_ack_request evtack_req;
3211         int retval = 0;
3212
3213         memset(&evtack_req, 0, sizeof(evtack_req));
3214         mutex_lock(&mrioc->init_cmds.mutex);
3215         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3216                 retval = -1;
3217                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3218                 mutex_unlock(&mrioc->init_cmds.mutex);
3219                 goto out;
3220         }
3221         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3222         mrioc->init_cmds.is_waiting = 1;
3223         mrioc->init_cmds.callback = NULL;
3224         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3225         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
3226         evtack_req.event = event;
3227         evtack_req.event_context = cpu_to_le32(event_ctx);
3228
3229         init_completion(&mrioc->init_cmds.done);
3230         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
3231             sizeof(evtack_req), 1);
3232         if (retval) {
3233                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3234                 goto out_unlock;
3235         }
3236         wait_for_completion_timeout(&mrioc->init_cmds.done,
3237             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3238         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3239                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3240                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3241                         mpi3mr_soft_reset_handler(mrioc,
3242                             MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
3243                 retval = -1;
3244                 goto out_unlock;
3245         }
3246         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3247             != MPI3_IOCSTATUS_SUCCESS) {
3248                 ioc_err(mrioc,
3249                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3250                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3251                     mrioc->init_cmds.ioc_loginfo);
3252                 retval = -1;
3253                 goto out_unlock;
3254         }
3255
3256 out_unlock:
3257         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3258         mutex_unlock(&mrioc->init_cmds.mutex);
3259 out:
3260         return retval;
3261 }
3262
3263 /**
3264  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3265  * @mrioc: Adapter instance reference
3266  *
3267  * Allocate chain buffers and set a bitmap to indicate free
3268  * chain buffers. Chain buffers are used to pass the SGE
3269  * information along with MPI3 SCSI IO requests for host I/O.
3270  *
3271  * Return: 0 on success, non-zero on failure
3272  */
3273 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3274 {
3275         int retval = 0;
3276         u32 sz, i;
3277         u16 num_chains;
3278
3279         if (mrioc->chain_sgl_list)
3280                 return retval;
3281
3282         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3283
3284         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
3285             | SHOST_DIX_TYPE1_PROTECTION
3286             | SHOST_DIX_TYPE2_PROTECTION
3287             | SHOST_DIX_TYPE3_PROTECTION))
3288                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
3289
3290         mrioc->chain_buf_count = num_chains;
3291         sz = sizeof(struct chain_element) * num_chains;
3292         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
3293         if (!mrioc->chain_sgl_list)
3294                 goto out_failed;
3295
3296         sz = MPI3MR_PAGE_SIZE_4K;
3297         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
3298             &mrioc->pdev->dev, sz, 16, 0);
3299         if (!mrioc->chain_buf_pool) {
3300                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3301                 goto out_failed;
3302         }
3303
3304         for (i = 0; i < num_chains; i++) {
3305                 mrioc->chain_sgl_list[i].addr =
3306                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
3307                     &mrioc->chain_sgl_list[i].dma_addr);
3308
3309                 if (!mrioc->chain_sgl_list[i].addr)
3310                         goto out_failed;
3311         }
3312         mrioc->chain_bitmap_sz = num_chains / 8;
3313         if (num_chains % 8)
3314                 mrioc->chain_bitmap_sz++;
3315         mrioc->chain_bitmap = kzalloc(mrioc->chain_bitmap_sz, GFP_KERNEL);
3316         if (!mrioc->chain_bitmap)
3317                 goto out_failed;
3318         return retval;
3319 out_failed:
3320         retval = -1;
3321         return retval;
3322 }
3323
3324 /**
3325  * mpi3mr_port_enable_complete - Mark port enable complete
3326  * @mrioc: Adapter instance reference
3327  * @drv_cmd: Internal command tracker
3328  *
3329  * Call back for asynchronous port enable request sets the
3330  * driver command to indicate port enable request is complete.
3331  *
3332  * Return: Nothing
3333  */
3334 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3335         struct mpi3mr_drv_cmd *drv_cmd)
3336 {
3337         drv_cmd->state = MPI3MR_CMD_NOTUSED;
3338         drv_cmd->callback = NULL;
3339         mrioc->scan_failed = drv_cmd->ioc_status;
3340         mrioc->scan_started = 0;
3341 }
3342
3343 /**
3344  * mpi3mr_issue_port_enable - Issue Port Enable
3345  * @mrioc: Adapter instance reference
3346  * @async: Flag to wait for completion or not
3347  *
3348  * Issue Port Enable MPI request through admin queue and if the
3349  * async flag is not set wait for the completion of the port
3350  * enable or time out.
3351  *
3352  * Return: 0 on success, non-zero on failures.
3353  */
3354 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3355 {
3356         struct mpi3_port_enable_request pe_req;
3357         int retval = 0;
3358         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
3359
3360         memset(&pe_req, 0, sizeof(pe_req));
3361         mutex_lock(&mrioc->init_cmds.mutex);
3362         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3363                 retval = -1;
3364                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3365                 mutex_unlock(&mrioc->init_cmds.mutex);
3366                 goto out;
3367         }
3368         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3369         if (async) {
3370                 mrioc->init_cmds.is_waiting = 0;
3371                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3372         } else {
3373                 mrioc->init_cmds.is_waiting = 1;
3374                 mrioc->init_cmds.callback = NULL;
3375                 init_completion(&mrioc->init_cmds.done);
3376         }
3377         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3378         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
3379
3380         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
3381         if (retval) {
3382                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3383                 goto out_unlock;
3384         }
3385         if (async) {
3386                 mutex_unlock(&mrioc->init_cmds.mutex);
3387                 goto out;
3388         }
3389
3390         wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
3391         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3392                 ioc_err(mrioc, "port enable timed out\n");
3393                 retval = -1;
3394                 mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
3395                 goto out_unlock;
3396         }
3397         mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
3398
3399 out_unlock:
3400         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3401         mutex_unlock(&mrioc->init_cmds.mutex);
3402 out:
3403         return retval;
3404 }
3405
3406 /* Protocol type to name mapper structure */
3407 static const struct {
3408         u8 protocol;
3409         char *name;
3410 } mpi3mr_protocols[] = {
3411         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3412         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3413         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3414 };
3415
3416 /* Capability to name mapper structure*/
3417 static const struct {
3418         u32 capability;
3419         char *name;
3420 } mpi3mr_capabilities[] = {
3421         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3422 };
3423
3424 /**
3425  * mpi3mr_print_ioc_info - Display controller information
3426  * @mrioc: Adapter instance reference
3427  *
3428  * Display controller personalit, capability, supported
3429  * protocols etc.
3430  *
3431  * Return: Nothing
3432  */
3433 static void
3434 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3435 {
3436         int i = 0, bytes_written = 0;
3437         char personality[16];
3438         char protocol[50] = {0};
3439         char capabilities[100] = {0};
3440         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3441
3442         switch (mrioc->facts.personality) {
3443         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3444                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3445                 break;
3446         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3447                 strncpy(personality, "RAID", sizeof(personality));
3448                 break;
3449         default:
3450                 strncpy(personality, "Unknown", sizeof(personality));
3451                 break;
3452         }
3453
3454         ioc_info(mrioc, "Running in %s Personality", personality);
3455
3456         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3457             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3458             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3459
3460         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3461                 if (mrioc->facts.protocol_flags &
3462                     mpi3mr_protocols[i].protocol) {
3463                         bytes_written += scnprintf(protocol + bytes_written,
3464                                     sizeof(protocol) - bytes_written, "%s%s",
3465                                     bytes_written ? "," : "",
3466                                     mpi3mr_protocols[i].name);
3467                 }
3468         }
3469
3470         bytes_written = 0;
3471         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3472                 if (mrioc->facts.protocol_flags &
3473                     mpi3mr_capabilities[i].capability) {
3474                         bytes_written += scnprintf(capabilities + bytes_written,
3475                                     sizeof(capabilities) - bytes_written, "%s%s",
3476                                     bytes_written ? "," : "",
3477                                     mpi3mr_capabilities[i].name);
3478                 }
3479         }
3480
3481         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3482                  protocol, capabilities);
3483 }
3484
3485 /**
3486  * mpi3mr_cleanup_resources - Free PCI resources
3487  * @mrioc: Adapter instance reference
3488  *
3489  * Unmap PCI device memory and disable PCI device.
3490  *
3491  * Return: 0 on success and non-zero on failure.
3492  */
3493 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3494 {
3495         struct pci_dev *pdev = mrioc->pdev;
3496
3497         mpi3mr_cleanup_isr(mrioc);
3498
3499         if (mrioc->sysif_regs) {
3500                 iounmap((void __iomem *)mrioc->sysif_regs);
3501                 mrioc->sysif_regs = NULL;
3502         }
3503
3504         if (pci_is_enabled(pdev)) {
3505                 if (mrioc->bars)
3506                         pci_release_selected_regions(pdev, mrioc->bars);
3507                 pci_disable_device(pdev);
3508         }
3509 }
3510
3511 /**
3512  * mpi3mr_setup_resources - Enable PCI resources
3513  * @mrioc: Adapter instance reference
3514  *
3515  * Enable PCI device memory, MSI-x registers and set DMA mask.
3516  *
3517  * Return: 0 on success and non-zero on failure.
3518  */
3519 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3520 {
3521         struct pci_dev *pdev = mrioc->pdev;
3522         u32 memap_sz = 0;
3523         int i, retval = 0, capb = 0;
3524         u16 message_control;
3525         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3526             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3527             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3528
3529         if (pci_enable_device_mem(pdev)) {
3530                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3531                 retval = -ENODEV;
3532                 goto out_failed;
3533         }
3534
3535         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3536         if (!capb) {
3537                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3538                 retval = -ENODEV;
3539                 goto out_failed;
3540         }
3541         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3542
3543         if (pci_request_selected_regions(pdev, mrioc->bars,
3544             mrioc->driver_name)) {
3545                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3546                 retval = -ENODEV;
3547                 goto out_failed;
3548         }
3549
3550         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3551                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3552                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3553                         memap_sz = pci_resource_len(pdev, i);
3554                         mrioc->sysif_regs =
3555                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3556                         break;
3557                 }
3558         }
3559
3560         pci_set_master(pdev);
3561
3562         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3563         if (retval) {
3564                 if (dma_mask != DMA_BIT_MASK(32)) {
3565                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3566                         dma_mask = DMA_BIT_MASK(32);
3567                         retval = dma_set_mask_and_coherent(&pdev->dev,
3568                             dma_mask);
3569                 }
3570                 if (retval) {
3571                         mrioc->dma_mask = 0;
3572                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3573                         goto out_failed;
3574                 }
3575         }
3576         mrioc->dma_mask = dma_mask;
3577
3578         if (!mrioc->sysif_regs) {
3579                 ioc_err(mrioc,
3580                     "Unable to map adapter memory or resource not found\n");
3581                 retval = -EINVAL;
3582                 goto out_failed;
3583         }
3584
3585         pci_read_config_word(pdev, capb + 2, &message_control);
3586         mrioc->msix_count = (message_control & 0x3FF) + 1;
3587
3588         pci_save_state(pdev);
3589
3590         pci_set_drvdata(pdev, mrioc->shost);
3591
3592         mpi3mr_ioc_disable_intr(mrioc);
3593
3594         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3595             (unsigned long long)mrioc->sysif_regs_phys,
3596             mrioc->sysif_regs, memap_sz);
3597         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3598             mrioc->msix_count);
3599
3600         if (!reset_devices && poll_queues > 0)
3601                 mrioc->requested_poll_qcount = min_t(int, poll_queues,
3602                                 mrioc->msix_count - 2);
3603         return retval;
3604
3605 out_failed:
3606         mpi3mr_cleanup_resources(mrioc);
3607         return retval;
3608 }
3609
3610 /**
3611  * mpi3mr_enable_events - Enable required events
3612  * @mrioc: Adapter instance reference
3613  *
3614  * This routine unmasks the events required by the driver by
3615  * sennding appropriate event mask bitmapt through an event
3616  * notification request.
3617  *
3618  * Return: 0 on success and non-zero on failure.
3619  */
3620 static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3621 {
3622         int retval = 0;
3623         u32  i;
3624
3625         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3626                 mrioc->event_masks[i] = -1;
3627
3628         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3629         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3630         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3631         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3632         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3633         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3634         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3635         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3636         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3637         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3638         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3639         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3640         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3641
3642         retval = mpi3mr_issue_event_notification(mrioc);
3643         if (retval)
3644                 ioc_err(mrioc, "failed to issue event notification %d\n",
3645                     retval);
3646         return retval;
3647 }
3648
3649 /**
3650  * mpi3mr_init_ioc - Initialize the controller
3651  * @mrioc: Adapter instance reference
3652  *
3653  * This the controller initialization routine, executed either
3654  * after soft reset or from pci probe callback.
3655  * Setup the required resources, memory map the controller
3656  * registers, create admin and operational reply queue pairs,
3657  * allocate required memory for reply pool, sense buffer pool,
3658  * issue IOC init request to the firmware, unmask the events and
3659  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3660  * volumes.
3661  *
3662  * Return: 0 on success and non-zero on failure.
3663  */
3664 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3665 {
3666         int retval = 0;
3667         u8 retry = 0;
3668         struct mpi3_ioc_facts_data facts_data;
3669
3670 retry_init:
3671         retval = mpi3mr_bring_ioc_ready(mrioc);
3672         if (retval) {
3673                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3674                     retval);
3675                 goto out_failed_noretry;
3676         }
3677
3678         retval = mpi3mr_setup_isr(mrioc, 1);
3679         if (retval) {
3680                 ioc_err(mrioc, "Failed to setup ISR error %d\n",
3681                     retval);
3682                 goto out_failed_noretry;
3683         }
3684
3685         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3686         if (retval) {
3687                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3688                     retval);
3689                 goto out_failed;
3690         }
3691
3692         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3693
3694         if (reset_devices)
3695                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3696                     MPI3MR_HOST_IOS_KDUMP);
3697
3698         mrioc->reply_sz = mrioc->facts.reply_sz;
3699
3700         retval = mpi3mr_check_reset_dma_mask(mrioc);
3701         if (retval) {
3702                 ioc_err(mrioc, "Resetting dma mask failed %d\n",
3703                     retval);
3704                 goto out_failed_noretry;
3705         }
3706
3707         mpi3mr_print_ioc_info(mrioc);
3708
3709         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3710         if (retval) {
3711                 ioc_err(mrioc,
3712                     "%s :Failed to allocated reply sense buffers %d\n",
3713                     __func__, retval);
3714                 goto out_failed_noretry;
3715         }
3716
3717         retval = mpi3mr_alloc_chain_bufs(mrioc);
3718         if (retval) {
3719                 ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3720                     retval);
3721                 goto out_failed_noretry;
3722         }
3723
3724         retval = mpi3mr_issue_iocinit(mrioc);
3725         if (retval) {
3726                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3727                     retval);
3728                 goto out_failed;
3729         }
3730
3731         retval = mpi3mr_print_pkg_ver(mrioc);
3732         if (retval) {
3733                 ioc_err(mrioc, "failed to get package version\n");
3734                 goto out_failed;
3735         }
3736
3737         retval = mpi3mr_setup_isr(mrioc, 0);
3738         if (retval) {
3739                 ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3740                     retval);
3741                 goto out_failed_noretry;
3742         }
3743
3744         retval = mpi3mr_create_op_queues(mrioc);
3745         if (retval) {
3746                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3747                     retval);
3748                 goto out_failed;
3749         }
3750
3751         if (!mrioc->pel_seqnum_virt) {
3752                 dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
3753                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
3754                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
3755                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
3756                     GFP_KERNEL);
3757                 if (!mrioc->pel_seqnum_virt) {
3758                         retval = -ENOMEM;
3759                         goto out_failed_noretry;
3760                 }
3761         }
3762
3763         retval = mpi3mr_enable_events(mrioc);
3764         if (retval) {
3765                 ioc_err(mrioc, "failed to enable events %d\n",
3766                     retval);
3767                 goto out_failed;
3768         }
3769
3770         ioc_info(mrioc, "controller initialization completed successfully\n");
3771         return retval;
3772 out_failed:
3773         if (retry < 2) {
3774                 retry++;
3775                 ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
3776                     retry);
3777                 mpi3mr_memset_buffers(mrioc);
3778                 goto retry_init;
3779         }
3780 out_failed_noretry:
3781         ioc_err(mrioc, "controller initialization failed\n");
3782         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
3783             MPI3MR_RESET_FROM_CTLR_CLEANUP);
3784         mrioc->unrecoverable = 1;
3785         return retval;
3786 }
3787
3788 /**
3789  * mpi3mr_reinit_ioc - Re-Initialize the controller
3790  * @mrioc: Adapter instance reference
3791  * @is_resume: Called from resume or reset path
3792  *
3793  * This the controller re-initialization routine, executed from
3794  * the soft reset handler or resume callback. Creates
3795  * operational reply queue pairs, allocate required memory for
3796  * reply pool, sense buffer pool, issue IOC init request to the
3797  * firmware, unmask the events and issue port enable to discover
3798  * SAS/SATA/NVMe devices and RAID volumes.
3799  *
3800  * Return: 0 on success and non-zero on failure.
3801  */
3802 int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
3803 {
3804         int retval = 0;
3805         u8 retry = 0;
3806         struct mpi3_ioc_facts_data facts_data;
3807
3808 retry_init:
3809         dprint_reset(mrioc, "bringing up the controller to ready state\n");
3810         retval = mpi3mr_bring_ioc_ready(mrioc);
3811         if (retval) {
3812                 ioc_err(mrioc, "failed to bring to ready state\n");
3813                 goto out_failed_noretry;
3814         }
3815
3816         if (is_resume) {
3817                 dprint_reset(mrioc, "setting up single ISR\n");
3818                 retval = mpi3mr_setup_isr(mrioc, 1);
3819                 if (retval) {
3820                         ioc_err(mrioc, "failed to setup ISR\n");
3821                         goto out_failed_noretry;
3822                 }
3823         } else
3824                 mpi3mr_ioc_enable_intr(mrioc);
3825
3826         dprint_reset(mrioc, "getting ioc_facts\n");
3827         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3828         if (retval) {
3829                 ioc_err(mrioc, "failed to get ioc_facts\n");
3830                 goto out_failed;
3831         }
3832
3833         dprint_reset(mrioc, "validating ioc_facts\n");
3834         retval = mpi3mr_revalidate_factsdata(mrioc);
3835         if (retval) {
3836                 ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
3837                 goto out_failed_noretry;
3838         }
3839
3840         mpi3mr_print_ioc_info(mrioc);
3841
3842         dprint_reset(mrioc, "sending ioc_init\n");
3843         retval = mpi3mr_issue_iocinit(mrioc);
3844         if (retval) {
3845                 ioc_err(mrioc, "failed to send ioc_init\n");
3846                 goto out_failed;
3847         }
3848
3849         dprint_reset(mrioc, "getting package version\n");
3850         retval = mpi3mr_print_pkg_ver(mrioc);
3851         if (retval) {
3852                 ioc_err(mrioc, "failed to get package version\n");
3853                 goto out_failed;
3854         }
3855
3856         if (is_resume) {
3857                 dprint_reset(mrioc, "setting up multiple ISR\n");
3858                 retval = mpi3mr_setup_isr(mrioc, 0);
3859                 if (retval) {
3860                         ioc_err(mrioc, "failed to re-setup ISR\n");
3861                         goto out_failed_noretry;
3862                 }
3863         }
3864
3865         dprint_reset(mrioc, "creating operational queue pairs\n");
3866         retval = mpi3mr_create_op_queues(mrioc);
3867         if (retval) {
3868                 ioc_err(mrioc, "failed to create operational queue pairs\n");
3869                 goto out_failed;
3870         }
3871
3872         if (!mrioc->pel_seqnum_virt) {
3873                 dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
3874                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
3875                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
3876                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
3877                     GFP_KERNEL);
3878                 if (!mrioc->pel_seqnum_virt) {
3879                         retval = -ENOMEM;
3880                         goto out_failed_noretry;
3881                 }
3882         }
3883
3884         if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
3885                 ioc_err(mrioc,
3886                     "cannot create minimum number of operational queues expected:%d created:%d\n",
3887                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
3888                 goto out_failed_noretry;
3889         }
3890
3891         dprint_reset(mrioc, "enabling events\n");
3892         retval = mpi3mr_enable_events(mrioc);
3893         if (retval) {
3894                 ioc_err(mrioc, "failed to enable events\n");
3895                 goto out_failed;
3896         }
3897
3898         ioc_info(mrioc, "sending port enable\n");
3899         retval = mpi3mr_issue_port_enable(mrioc, 0);
3900         if (retval) {
3901                 ioc_err(mrioc, "failed to issue port enable\n");
3902                 goto out_failed;
3903         }
3904
3905         ioc_info(mrioc, "controller %s completed successfully\n",
3906             (is_resume)?"resume":"re-initialization");
3907         return retval;
3908 out_failed:
3909         if (retry < 2) {
3910                 retry++;
3911                 ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
3912                     (is_resume)?"resume":"re-initialization", retry);
3913                 mpi3mr_memset_buffers(mrioc);
3914                 goto retry_init;
3915         }
3916 out_failed_noretry:
3917         ioc_err(mrioc, "controller %s is failed\n",
3918             (is_resume)?"resume":"re-initialization");
3919         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
3920             MPI3MR_RESET_FROM_CTLR_CLEANUP);
3921         mrioc->unrecoverable = 1;
3922         return retval;
3923 }
3924
3925 /**
3926  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
3927  *                                      segments
3928  * @mrioc: Adapter instance reference
3929  * @qidx: Operational reply queue index
3930  *
3931  * Return: Nothing.
3932  */
3933 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3934 {
3935         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
3936         struct segments *segments;
3937         int i, size;
3938
3939         if (!op_reply_q->q_segments)
3940                 return;
3941
3942         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
3943         segments = op_reply_q->q_segments;
3944         for (i = 0; i < op_reply_q->num_segments; i++)
3945                 memset(segments[i].segment, 0, size);
3946 }
3947
3948 /**
3949  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
3950  *                                      segments
3951  * @mrioc: Adapter instance reference
3952  * @qidx: Operational request queue index
3953  *
3954  * Return: Nothing.
3955  */
3956 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3957 {
3958         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
3959         struct segments *segments;
3960         int i, size;
3961
3962         if (!op_req_q->q_segments)
3963                 return;
3964
3965         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
3966         segments = op_req_q->q_segments;
3967         for (i = 0; i < op_req_q->num_segments; i++)
3968                 memset(segments[i].segment, 0, size);
3969 }
3970
3971 /**
3972  * mpi3mr_memset_buffers - memset memory for a controller
3973  * @mrioc: Adapter instance reference
3974  *
3975  * clear all the memory allocated for a controller, typically
3976  * called post reset to reuse the memory allocated during the
3977  * controller init.
3978  *
3979  * Return: Nothing.
3980  */
3981 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
3982 {
3983         u16 i;
3984
3985         mrioc->change_count = 0;
3986         mrioc->active_poll_qcount = 0;
3987         mrioc->default_qcount = 0;
3988         if (mrioc->admin_req_base)
3989                 memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
3990         if (mrioc->admin_reply_base)
3991                 memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
3992
3993         if (mrioc->init_cmds.reply) {
3994                 memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
3995                 memset(mrioc->bsg_cmds.reply, 0,
3996                     sizeof(*mrioc->bsg_cmds.reply));
3997                 memset(mrioc->host_tm_cmds.reply, 0,
3998                     sizeof(*mrioc->host_tm_cmds.reply));
3999                 memset(mrioc->pel_cmds.reply, 0,
4000                     sizeof(*mrioc->pel_cmds.reply));
4001                 memset(mrioc->pel_abort_cmd.reply, 0,
4002                     sizeof(*mrioc->pel_abort_cmd.reply));
4003                 for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4004                         memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4005                             sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4006                 for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
4007                         memset(mrioc->evtack_cmds[i].reply, 0,
4008                             sizeof(*mrioc->evtack_cmds[i].reply));
4009                 memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
4010                 memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
4011                 memset(mrioc->evtack_cmds_bitmap, 0,
4012                     mrioc->evtack_cmds_bitmap_sz);
4013         }
4014
4015         for (i = 0; i < mrioc->num_queues; i++) {
4016                 mrioc->op_reply_qinfo[i].qid = 0;
4017                 mrioc->op_reply_qinfo[i].ci = 0;
4018                 mrioc->op_reply_qinfo[i].num_replies = 0;
4019                 mrioc->op_reply_qinfo[i].ephase = 0;
4020                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
4021                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
4022                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
4023
4024                 mrioc->req_qinfo[i].ci = 0;
4025                 mrioc->req_qinfo[i].pi = 0;
4026                 mrioc->req_qinfo[i].num_requests = 0;
4027                 mrioc->req_qinfo[i].qid = 0;
4028                 mrioc->req_qinfo[i].reply_qid = 0;
4029                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4030                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
4031         }
4032 }
4033
4034 /**
4035  * mpi3mr_free_mem - Free memory allocated for a controller
4036  * @mrioc: Adapter instance reference
4037  *
4038  * Free all the memory allocated for a controller.
4039  *
4040  * Return: Nothing.
4041  */
4042 void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4043 {
4044         u16 i;
4045         struct mpi3mr_intr_info *intr_info;
4046
4047         if (mrioc->sense_buf_pool) {
4048                 if (mrioc->sense_buf)
4049                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
4050                             mrioc->sense_buf_dma);
4051                 dma_pool_destroy(mrioc->sense_buf_pool);
4052                 mrioc->sense_buf = NULL;
4053                 mrioc->sense_buf_pool = NULL;
4054         }
4055         if (mrioc->sense_buf_q_pool) {
4056                 if (mrioc->sense_buf_q)
4057                         dma_pool_free(mrioc->sense_buf_q_pool,
4058                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
4059                 dma_pool_destroy(mrioc->sense_buf_q_pool);
4060                 mrioc->sense_buf_q = NULL;
4061                 mrioc->sense_buf_q_pool = NULL;
4062         }
4063
4064         if (mrioc->reply_buf_pool) {
4065                 if (mrioc->reply_buf)
4066                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
4067                             mrioc->reply_buf_dma);
4068                 dma_pool_destroy(mrioc->reply_buf_pool);
4069                 mrioc->reply_buf = NULL;
4070                 mrioc->reply_buf_pool = NULL;
4071         }
4072         if (mrioc->reply_free_q_pool) {
4073                 if (mrioc->reply_free_q)
4074                         dma_pool_free(mrioc->reply_free_q_pool,
4075                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
4076                 dma_pool_destroy(mrioc->reply_free_q_pool);
4077                 mrioc->reply_free_q = NULL;
4078                 mrioc->reply_free_q_pool = NULL;
4079         }
4080
4081         for (i = 0; i < mrioc->num_op_req_q; i++)
4082                 mpi3mr_free_op_req_q_segments(mrioc, i);
4083
4084         for (i = 0; i < mrioc->num_op_reply_q; i++)
4085                 mpi3mr_free_op_reply_q_segments(mrioc, i);
4086
4087         for (i = 0; i < mrioc->intr_info_count; i++) {
4088                 intr_info = mrioc->intr_info + i;
4089                 intr_info->op_reply_q = NULL;
4090         }
4091
4092         kfree(mrioc->req_qinfo);
4093         mrioc->req_qinfo = NULL;
4094         mrioc->num_op_req_q = 0;
4095
4096         kfree(mrioc->op_reply_qinfo);
4097         mrioc->op_reply_qinfo = NULL;
4098         mrioc->num_op_reply_q = 0;
4099
4100         kfree(mrioc->init_cmds.reply);
4101         mrioc->init_cmds.reply = NULL;
4102
4103         kfree(mrioc->bsg_cmds.reply);
4104         mrioc->bsg_cmds.reply = NULL;
4105
4106         kfree(mrioc->host_tm_cmds.reply);
4107         mrioc->host_tm_cmds.reply = NULL;
4108
4109         kfree(mrioc->pel_cmds.reply);
4110         mrioc->pel_cmds.reply = NULL;
4111
4112         kfree(mrioc->pel_abort_cmd.reply);
4113         mrioc->pel_abort_cmd.reply = NULL;
4114
4115         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4116                 kfree(mrioc->evtack_cmds[i].reply);
4117                 mrioc->evtack_cmds[i].reply = NULL;
4118         }
4119
4120         kfree(mrioc->removepend_bitmap);
4121         mrioc->removepend_bitmap = NULL;
4122
4123         kfree(mrioc->devrem_bitmap);
4124         mrioc->devrem_bitmap = NULL;
4125
4126         kfree(mrioc->evtack_cmds_bitmap);
4127         mrioc->evtack_cmds_bitmap = NULL;
4128
4129         kfree(mrioc->chain_bitmap);
4130         mrioc->chain_bitmap = NULL;
4131
4132         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4133                 kfree(mrioc->dev_rmhs_cmds[i].reply);
4134                 mrioc->dev_rmhs_cmds[i].reply = NULL;
4135         }
4136
4137         if (mrioc->chain_buf_pool) {
4138                 for (i = 0; i < mrioc->chain_buf_count; i++) {
4139                         if (mrioc->chain_sgl_list[i].addr) {
4140                                 dma_pool_free(mrioc->chain_buf_pool,
4141                                     mrioc->chain_sgl_list[i].addr,
4142                                     mrioc->chain_sgl_list[i].dma_addr);
4143                                 mrioc->chain_sgl_list[i].addr = NULL;
4144                         }
4145                 }
4146                 dma_pool_destroy(mrioc->chain_buf_pool);
4147                 mrioc->chain_buf_pool = NULL;
4148         }
4149
4150         kfree(mrioc->chain_sgl_list);
4151         mrioc->chain_sgl_list = NULL;
4152
4153         if (mrioc->admin_reply_base) {
4154                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
4155                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
4156                 mrioc->admin_reply_base = NULL;
4157         }
4158         if (mrioc->admin_req_base) {
4159                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
4160                     mrioc->admin_req_base, mrioc->admin_req_dma);
4161                 mrioc->admin_req_base = NULL;
4162         }
4163
4164         if (mrioc->pel_seqnum_virt) {
4165                 dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,
4166                     mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);
4167                 mrioc->pel_seqnum_virt = NULL;
4168         }
4169
4170         kfree(mrioc->logdata_buf);
4171         mrioc->logdata_buf = NULL;
4172
4173 }
4174
4175 /**
4176  * mpi3mr_issue_ioc_shutdown - shutdown controller
4177  * @mrioc: Adapter instance reference
4178  *
4179  * Send shutodwn notification to the controller and wait for the
4180  * shutdown_timeout for it to be completed.
4181  *
4182  * Return: Nothing.
4183  */
4184 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4185 {
4186         u32 ioc_config, ioc_status;
4187         u8 retval = 1;
4188         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
4189
4190         ioc_info(mrioc, "Issuing shutdown Notification\n");
4191         if (mrioc->unrecoverable) {
4192                 ioc_warn(mrioc,
4193                     "IOC is unrecoverable shutdown is not issued\n");
4194                 return;
4195         }
4196         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4197         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4198             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
4199                 ioc_info(mrioc, "shutdown already in progress\n");
4200                 return;
4201         }
4202
4203         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4204         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
4205         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
4206
4207         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
4208
4209         if (mrioc->facts.shutdown_timeout)
4210                 timeout = mrioc->facts.shutdown_timeout * 10;
4211
4212         do {
4213                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4214                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4215                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
4216                         retval = 0;
4217                         break;
4218                 }
4219                 msleep(100);
4220         } while (--timeout);
4221
4222         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4223         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4224
4225         if (retval) {
4226                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4227                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
4228                         ioc_warn(mrioc,
4229                             "shutdown still in progress after timeout\n");
4230         }
4231
4232         ioc_info(mrioc,
4233             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
4234             (!retval) ? "successful" : "failed", ioc_status,
4235             ioc_config);
4236 }
4237
4238 /**
4239  * mpi3mr_cleanup_ioc - Cleanup controller
4240  * @mrioc: Adapter instance reference
4241  *
4242  * controller cleanup handler, Message unit reset or soft reset
4243  * and shutdown notification is issued to the controller.
4244  *
4245  * Return: Nothing.
4246  */
4247 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4248 {
4249         enum mpi3mr_iocstate ioc_state;
4250
4251         dprint_exit(mrioc, "cleaning up the controller\n");
4252         mpi3mr_ioc_disable_intr(mrioc);
4253
4254         ioc_state = mpi3mr_get_iocstate(mrioc);
4255
4256         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4257             (ioc_state == MRIOC_STATE_READY)) {
4258                 if (mpi3mr_issue_and_process_mur(mrioc,
4259                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
4260                         mpi3mr_issue_reset(mrioc,
4261                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
4262                             MPI3MR_RESET_FROM_MUR_FAILURE);
4263                 mpi3mr_issue_ioc_shutdown(mrioc);
4264         }
4265         dprint_exit(mrioc, "controller cleanup completed\n");
4266 }
4267
4268 /**
4269  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4270  * @mrioc: Adapter instance reference
4271  * @cmdptr: Internal command tracker
4272  *
4273  * Complete an internal driver commands with state indicating it
4274  * is completed due to reset.
4275  *
4276  * Return: Nothing.
4277  */
4278 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4279         struct mpi3mr_drv_cmd *cmdptr)
4280 {
4281         if (cmdptr->state & MPI3MR_CMD_PENDING) {
4282                 cmdptr->state |= MPI3MR_CMD_RESET;
4283                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
4284                 if (cmdptr->is_waiting) {
4285                         complete(&cmdptr->done);
4286                         cmdptr->is_waiting = 0;
4287                 } else if (cmdptr->callback)
4288                         cmdptr->callback(mrioc, cmdptr);
4289         }
4290 }
4291
4292 /**
4293  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4294  * @mrioc: Adapter instance reference
4295  *
4296  * Flush all internal driver commands post reset
4297  *
4298  * Return: Nothing.
4299  */
4300 static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4301 {
4302         struct mpi3mr_drv_cmd *cmdptr;
4303         u8 i;
4304
4305         cmdptr = &mrioc->init_cmds;
4306         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4307         cmdptr = &mrioc->bsg_cmds;
4308         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4309         cmdptr = &mrioc->host_tm_cmds;
4310         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4311
4312         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4313                 cmdptr = &mrioc->dev_rmhs_cmds[i];
4314                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4315         }
4316
4317         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4318                 cmdptr = &mrioc->evtack_cmds[i];
4319                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4320         }
4321
4322         cmdptr = &mrioc->pel_cmds;
4323         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4324
4325         cmdptr = &mrioc->pel_abort_cmd;
4326         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4327
4328 }
4329
4330 /**
4331  * mpi3mr_pel_wait_post - Issue PEL Wait
4332  * @mrioc: Adapter instance reference
4333  * @drv_cmd: Internal command tracker
4334  *
4335  * Issue PEL Wait MPI request through admin queue and return.
4336  *
4337  * Return: Nothing.
4338  */
4339 static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4340         struct mpi3mr_drv_cmd *drv_cmd)
4341 {
4342         struct mpi3_pel_req_action_wait pel_wait;
4343
4344         mrioc->pel_abort_requested = false;
4345
4346         memset(&pel_wait, 0, sizeof(pel_wait));
4347         drv_cmd->state = MPI3MR_CMD_PENDING;
4348         drv_cmd->is_waiting = 0;
4349         drv_cmd->callback = mpi3mr_pel_wait_complete;
4350         drv_cmd->ioc_status = 0;
4351         drv_cmd->ioc_loginfo = 0;
4352         pel_wait.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4353         pel_wait.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4354         pel_wait.action = MPI3_PEL_ACTION_WAIT;
4355         pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4356         pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4357         pel_wait.class = cpu_to_le16(mrioc->pel_class);
4358         pel_wait.wait_time = MPI3_PEL_WAITTIME_INFINITE_WAIT;
4359         dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4360             mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4361
4362         if (mpi3mr_admin_request_post(mrioc, &pel_wait, sizeof(pel_wait), 0)) {
4363                 dprint_bsg_err(mrioc,
4364                             "Issuing PELWait: Admin post failed\n");
4365                 drv_cmd->state = MPI3MR_CMD_NOTUSED;
4366                 drv_cmd->callback = NULL;
4367                 drv_cmd->retry_count = 0;
4368                 mrioc->pel_enabled = false;
4369         }
4370 }
4371
4372 /**
4373  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4374  * @mrioc: Adapter instance reference
4375  * @drv_cmd: Internal command tracker
4376  *
4377  * Issue PEL get sequence number MPI request through admin queue
4378  * and return.
4379  *
4380  * Return: 0 on success, non-zero on failure.
4381  */
4382 int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4383         struct mpi3mr_drv_cmd *drv_cmd)
4384 {
4385         struct mpi3_pel_req_action_get_sequence_numbers pel_getseq_req;
4386         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
4387         int retval = 0;
4388
4389         memset(&pel_getseq_req, 0, sizeof(pel_getseq_req));
4390         mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4391         mrioc->pel_cmds.is_waiting = 0;
4392         mrioc->pel_cmds.ioc_status = 0;
4393         mrioc->pel_cmds.ioc_loginfo = 0;
4394         mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4395         pel_getseq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4396         pel_getseq_req.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4397         pel_getseq_req.action = MPI3_PEL_ACTION_GET_SEQNUM;
4398         mpi3mr_add_sg_single(&pel_getseq_req.sgl, sgl_flags,
4399             mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);
4400
4401         retval = mpi3mr_admin_request_post(mrioc, &pel_getseq_req,
4402                         sizeof(pel_getseq_req), 0);
4403         if (retval) {
4404                 if (drv_cmd) {
4405                         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4406                         drv_cmd->callback = NULL;
4407                         drv_cmd->retry_count = 0;
4408                 }
4409                 mrioc->pel_enabled = false;
4410         }
4411
4412         return retval;
4413 }
4414
4415 /**
4416  * mpi3mr_pel_wait_complete - PELWait Completion callback
4417  * @mrioc: Adapter instance reference
4418  * @drv_cmd: Internal command tracker
4419  *
4420  * This is a callback handler for the PELWait request and
4421  * firmware completes a PELWait request when it is aborted or a
4422  * new PEL entry is available. This sends AEN to the application
4423  * and if the PELwait completion is not due to PELAbort then
4424  * this will send a request for new PEL Sequence number
4425  *
4426  * Return: Nothing.
4427  */
4428 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4429         struct mpi3mr_drv_cmd *drv_cmd)
4430 {
4431         struct mpi3_pel_reply *pel_reply = NULL;
4432         u16 ioc_status, pe_log_status;
4433         bool do_retry = false;
4434
4435         if (drv_cmd->state & MPI3MR_CMD_RESET)
4436                 goto cleanup_drv_cmd;
4437
4438         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4439         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4440                 ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4441                         __func__, ioc_status, drv_cmd->ioc_loginfo);
4442                 dprint_bsg_err(mrioc,
4443                     "pel_wait: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4444                     ioc_status, drv_cmd->ioc_loginfo);
4445                 do_retry = true;
4446         }
4447
4448         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4449                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4450
4451         if (!pel_reply) {
4452                 dprint_bsg_err(mrioc,
4453                     "pel_wait: failed due to no reply\n");
4454                 goto out_failed;
4455         }
4456
4457         pe_log_status = le16_to_cpu(pel_reply->pe_log_status);
4458         if ((pe_log_status != MPI3_PEL_STATUS_SUCCESS) &&
4459             (pe_log_status != MPI3_PEL_STATUS_ABORTED)) {
4460                 ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4461                         __func__, pe_log_status);
4462                 dprint_bsg_err(mrioc,
4463                     "pel_wait: failed due to pel_log_status(0x%04x)\n",
4464                     pe_log_status);
4465                 do_retry = true;
4466         }
4467
4468         if (do_retry) {
4469                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4470                         drv_cmd->retry_count++;
4471                         dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4472                             drv_cmd->retry_count);
4473                         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4474                         return;
4475                 }
4476                 dprint_bsg_err(mrioc,
4477                     "pel_wait: failed after all retries(%d)\n",
4478                     drv_cmd->retry_count);
4479                 goto out_failed;
4480         }
4481         atomic64_inc(&event_counter);
4482         if (!mrioc->pel_abort_requested) {
4483                 mrioc->pel_cmds.retry_count = 0;
4484                 mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);
4485         }
4486
4487         return;
4488 out_failed:
4489         mrioc->pel_enabled = false;
4490 cleanup_drv_cmd:
4491         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4492         drv_cmd->callback = NULL;
4493         drv_cmd->retry_count = 0;
4494 }
4495
4496 /**
4497  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4498  * @mrioc: Adapter instance reference
4499  * @drv_cmd: Internal command tracker
4500  *
4501  * This is a callback handler for the PEL get sequence number
4502  * request and a new PEL wait request will be issued to the
4503  * firmware from this
4504  *
4505  * Return: Nothing.
4506  */
4507 void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4508         struct mpi3mr_drv_cmd *drv_cmd)
4509 {
4510         struct mpi3_pel_reply *pel_reply = NULL;
4511         struct mpi3_pel_seq *pel_seqnum_virt;
4512         u16 ioc_status;
4513         bool do_retry = false;
4514
4515         pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4516
4517         if (drv_cmd->state & MPI3MR_CMD_RESET)
4518                 goto cleanup_drv_cmd;
4519
4520         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4521         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4522                 dprint_bsg_err(mrioc,
4523                     "pel_get_seqnum: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4524                     ioc_status, drv_cmd->ioc_loginfo);
4525                 do_retry = true;
4526         }
4527
4528         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4529                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4530         if (!pel_reply) {
4531                 dprint_bsg_err(mrioc,
4532                     "pel_get_seqnum: failed due to no reply\n");
4533                 goto out_failed;
4534         }
4535
4536         if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {
4537                 dprint_bsg_err(mrioc,
4538                     "pel_get_seqnum: failed due to pel_log_status(0x%04x)\n",
4539                     le16_to_cpu(pel_reply->pe_log_status));
4540                 do_retry = true;
4541         }
4542
4543         if (do_retry) {
4544                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4545                         drv_cmd->retry_count++;
4546                         dprint_bsg_err(mrioc,
4547                             "pel_get_seqnum: retrying(%d)\n",
4548                             drv_cmd->retry_count);
4549                         mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4550                         return;
4551                 }
4552
4553                 dprint_bsg_err(mrioc,
4554                     "pel_get_seqnum: failed after all retries(%d)\n",
4555                     drv_cmd->retry_count);
4556                 goto out_failed;
4557         }
4558         mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4559         drv_cmd->retry_count = 0;
4560         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4561
4562         return;
4563 out_failed:
4564         mrioc->pel_enabled = false;
4565 cleanup_drv_cmd:
4566         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4567         drv_cmd->callback = NULL;
4568         drv_cmd->retry_count = 0;
4569 }
4570
4571 /**
4572  * mpi3mr_soft_reset_handler - Reset the controller
4573  * @mrioc: Adapter instance reference
4574  * @reset_reason: Reset reason code
4575  * @snapdump: Flag to generate snapdump in firmware or not
4576  *
4577  * This is an handler for recovering controller by issuing soft
4578  * reset are diag fault reset.  This is a blocking function and
4579  * when one reset is executed if any other resets they will be
4580  * blocked. All BSG requests will be blocked during the reset. If
4581  * controller reset is successful then the controller will be
4582  * reinitalized, otherwise the controller will be marked as not
4583  * recoverable
4584  *
4585  * In snapdump bit is set, the controller is issued with diag
4586  * fault reset so that the firmware can create a snap dump and
4587  * post that the firmware will result in F000 fault and the
4588  * driver will issue soft reset to recover from that.
4589  *
4590  * Return: 0 on success, non-zero on failure.
4591  */
4592 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4593         u32 reset_reason, u8 snapdump)
4594 {
4595         int retval = 0, i;
4596         unsigned long flags;
4597         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
4598
4599         /* Block the reset handler until diag save in progress*/
4600         dprint_reset(mrioc,
4601             "soft_reset_handler: check and block on diagsave_timeout(%d)\n",
4602             mrioc->diagsave_timeout);
4603         while (mrioc->diagsave_timeout)
4604                 ssleep(1);
4605         /*
4606          * Block new resets until the currently executing one is finished and
4607          * return the status of the existing reset for all blocked resets
4608          */
4609         dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4610         if (!mutex_trylock(&mrioc->reset_mutex)) {
4611                 ioc_info(mrioc,
4612                     "controller reset triggered by %s is blocked due to another reset in progress\n",
4613                     mpi3mr_reset_rc_name(reset_reason));
4614                 do {
4615                         ssleep(1);
4616                 } while (mrioc->reset_in_progress == 1);
4617                 ioc_info(mrioc,
4618                     "returning previous reset result(%d) for the reset triggered by %s\n",
4619                     mrioc->prev_reset_result,
4620                     mpi3mr_reset_rc_name(reset_reason));
4621                 return mrioc->prev_reset_result;
4622         }
4623         ioc_info(mrioc, "controller reset is triggered by %s\n",
4624             mpi3mr_reset_rc_name(reset_reason));
4625
4626         mrioc->reset_in_progress = 1;
4627         mrioc->stop_bsgs = 1;
4628         mrioc->prev_reset_result = -1;
4629
4630         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
4631             (reset_reason != MPI3MR_RESET_FROM_FIRMWARE) &&
4632             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
4633                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
4634                         mrioc->event_masks[i] = -1;
4635
4636                 dprint_reset(mrioc, "soft_reset_handler: masking events\n");
4637                 mpi3mr_issue_event_notification(mrioc);
4638         }
4639
4640         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
4641
4642         mpi3mr_ioc_disable_intr(mrioc);
4643
4644         if (snapdump) {
4645                 mpi3mr_set_diagsave(mrioc);
4646                 retval = mpi3mr_issue_reset(mrioc,
4647                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4648                 if (!retval) {
4649                         do {
4650                                 host_diagnostic =
4651                                     readl(&mrioc->sysif_regs->host_diagnostic);
4652                                 if (!(host_diagnostic &
4653                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
4654                                         break;
4655                                 msleep(100);
4656                         } while (--timeout);
4657                 }
4658         }
4659
4660         retval = mpi3mr_issue_reset(mrioc,
4661             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
4662         if (retval) {
4663                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
4664                 goto out;
4665         }
4666
4667         mpi3mr_flush_delayed_cmd_lists(mrioc);
4668         mpi3mr_flush_drv_cmds(mrioc);
4669         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
4670         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
4671         memset(mrioc->evtack_cmds_bitmap, 0, mrioc->evtack_cmds_bitmap_sz);
4672         mpi3mr_flush_host_io(mrioc);
4673         mpi3mr_cleanup_fwevt_list(mrioc);
4674         mpi3mr_invalidate_devhandles(mrioc);
4675         if (mrioc->prepare_for_reset) {
4676                 mrioc->prepare_for_reset = 0;
4677                 mrioc->prepare_for_reset_timeout_counter = 0;
4678         }
4679         mpi3mr_memset_buffers(mrioc);
4680         retval = mpi3mr_reinit_ioc(mrioc, 0);
4681         if (retval) {
4682                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
4683                     mrioc->name, reset_reason);
4684                 goto out;
4685         }
4686         ssleep(10);
4687
4688 out:
4689         if (!retval) {
4690                 mrioc->diagsave_timeout = 0;
4691                 mrioc->reset_in_progress = 0;
4692                 mrioc->pel_abort_requested = 0;
4693                 if (mrioc->pel_enabled) {
4694                         mrioc->pel_cmds.retry_count = 0;
4695                         mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);
4696                 }
4697
4698                 mpi3mr_rfresh_tgtdevs(mrioc);
4699                 mrioc->ts_update_counter = 0;
4700                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
4701                 if (mrioc->watchdog_work_q)
4702                         queue_delayed_work(mrioc->watchdog_work_q,
4703                             &mrioc->watchdog_work,
4704                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
4705                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
4706                 mrioc->stop_bsgs = 0;
4707                 if (mrioc->pel_enabled)
4708                         atomic64_inc(&event_counter);
4709         } else {
4710                 mpi3mr_issue_reset(mrioc,
4711                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4712                 mrioc->unrecoverable = 1;
4713                 mrioc->reset_in_progress = 0;
4714                 retval = -1;
4715         }
4716         mrioc->prev_reset_result = retval;
4717         mutex_unlock(&mrioc->reset_mutex);
4718         ioc_info(mrioc, "controller reset is %s\n",
4719             ((retval == 0) ? "successful" : "failed"));
4720         return retval;
4721 }