drivers/scsi/mpt3sas/mpt3sas_base.c

   1 /*
   2  * This is the Fusion MPT base driver providing common API layer interface
   3  * for access to MPT (Message Passing Technology) firmware.
   4  *
   5  * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
   6  * Copyright (C) 2012-2014  LSI Corporation
   7  * Copyright (C) 2013-2014 Avago Technologies
   8  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * NO WARRANTY
  21  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  22  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  23  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  24  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  25  * solely responsible for determining the appropriateness of using and
  26  * distributing the Program and assumes all risks associated with its
  27  * exercise of rights under this Agreement, including but not limited to
  28  * the risks and costs of program errors, damage to or loss of data,
  29  * programs or equipment, and unavailability or interruption of operations.
  30
  31  * DISCLAIMER OF LIABILITY
  32  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  33  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  37  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  38  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  39
  40  * You should have received a copy of the GNU General Public License
  41  * along with this program; if not, write to the Free Software
  42  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  43  * USA.
  44  */
  45
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/errno.h>
  49 #include <linux/init.h>
  50 #include <linux/slab.h>
  51 #include <linux/types.h>
  52 #include <linux/pci.h>
  53 #include <linux/kdev_t.h>
  54 #include <linux/blkdev.h>
  55 #include <linux/delay.h>
  56 #include <linux/interrupt.h>
  57 #include <linux/dma-mapping.h>
  58 #include <linux/io.h>
  59 #include <linux/time.h>
  60 #include <linux/ktime.h>
  61 #include <linux/kthread.h>
  62 #include <asm/page.h>        /* To get host page size per arch */
  63 #include <linux/aer.h>
  64
  65
  66 #include "mpt3sas_base.h"
  67
  68 static MPT_CALLBACK     mpt_callbacks[MPT_MAX_CALLBACKS];
  69
  70
  71 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
  72
  73  /* maximum controller queue depth */
  74 #define MAX_HBA_QUEUE_DEPTH     30000
  75 #define MAX_CHAIN_DEPTH         100000
  76 static int max_queue_depth = -1;
  77 module_param(max_queue_depth, int, 0444);
  78 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
  79
  80 static int max_sgl_entries = -1;
  81 module_param(max_sgl_entries, int, 0444);
  82 MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
  83
  84 static int msix_disable = -1;
  85 module_param(msix_disable, int, 0444);
  86 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
  87
  88 static int smp_affinity_enable = 1;
  89 module_param(smp_affinity_enable, int, 0444);
  90 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
  91
  92 static int max_msix_vectors = -1;
  93 module_param(max_msix_vectors, int, 0444);
  94 MODULE_PARM_DESC(max_msix_vectors,
  95         " max msix vectors");
  96
  97 static int irqpoll_weight = -1;
  98 module_param(irqpoll_weight, int, 0444);
  99 MODULE_PARM_DESC(irqpoll_weight,
 100         "irq poll weight (default= one fourth of HBA queue depth)");
 101
 102 static int mpt3sas_fwfault_debug;
 103 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
 104         " enable detection of firmware fault and halt firmware - (default=0)");
 105
 106 static int perf_mode = -1;
 107 module_param(perf_mode, int, 0444);
 108 MODULE_PARM_DESC(perf_mode,
 109         "Performance mode (only for Aero/Sea Generation), options:\n\t\t"
 110         "0 - balanced: high iops mode is enabled &\n\t\t"
 111         "interrupt coalescing is enabled only on high iops queues,\n\t\t"
 112         "1 - iops: high iops mode is disabled &\n\t\t"
 113         "interrupt coalescing is enabled on all queues,\n\t\t"
 114         "2 - latency: high iops mode is disabled &\n\t\t"
 115         "interrupt coalescing is enabled on all queues with timeout value 0xA,\n"
 116         "\t\tdefault - default perf_mode is 'balanced'"
 117         );
 118
 119 enum mpt3sas_perf_mode {
 120         MPT_PERF_MODE_DEFAULT   = -1,
 121         MPT_PERF_MODE_BALANCED  = 0,
 122         MPT_PERF_MODE_IOPS      = 1,
 123         MPT_PERF_MODE_LATENCY   = 2,
 124 };
 125
 126 static int
 127 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc,
 128                 u32 ioc_state, int timeout);
 129 static int
 130 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
 131 static void
 132 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc);
 133
 134 /**
 135  * mpt3sas_base_check_cmd_timeout - Function
 136  *              to check timeout and command termination due
 137  *              to Host reset.
 138  *
 139  * @ioc:        per adapter object.
 140  * @status:     Status of issued command.
 141  * @mpi_request:mf request pointer.
 142  * @sz:         size of buffer.
 143  *
 144  * @Returns - 1/0 Reset to be done or Not
 145  */
 146 u8
 147 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc,
 148                 u8 status, void *mpi_request, int sz)
 149 {
 150         u8 issue_reset = 0;
 151
 152         if (!(status & MPT3_CMD_RESET))
 153                 issue_reset = 1;
 154
 155         ioc_err(ioc, "Command %s\n",
 156                 issue_reset == 0 ? "terminated due to Host Reset" : "Timeout");
 157         _debug_dump_mf(mpi_request, sz);
 158
 159         return issue_reset;
 160 }
 161
 162 /**
 163  * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
 164  * @val: ?
 165  * @kp: ?
 166  *
 167  * Return: ?
 168  */
 169 static int
 170 _scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp)
 171 {
 172         int ret = param_set_int(val, kp);
 173         struct MPT3SAS_ADAPTER *ioc;
 174
 175         if (ret)
 176                 return ret;
 177
 178         /* global ioc spinlock to protect controller list on list operations */
 179         pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
 180         spin_lock(&gioc_lock);
 181         list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
 182                 ioc->fwfault_debug = mpt3sas_fwfault_debug;
 183         spin_unlock(&gioc_lock);
 184         return 0;
 185 }
 186 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
 187         param_get_int, &mpt3sas_fwfault_debug, 0644);
 188
 189 /**
 190  * _base_readl_aero - retry readl for max three times.
 191  * @addr: MPT Fusion system interface register address
 192  *
 193  * Retry the readl() for max three times if it gets zero value
 194  * while reading the system interface register.
 195  */
 196 static inline u32
 197 _base_readl_aero(const volatile void __iomem *addr)
 198 {
 199         u32 i = 0, ret_val;
 200
 201         do {
 202                 ret_val = readl(addr);
 203                 i++;
 204         } while (ret_val == 0 && i < 3);
 205
 206         return ret_val;
 207 }
 208
 209 static inline u32
 210 _base_readl(const volatile void __iomem *addr)
 211 {
 212         return readl(addr);
 213 }
 214
 215 /**
 216  * _base_clone_reply_to_sys_mem - copies reply to reply free iomem
 217  *                                in BAR0 space.
 218  *
 219  * @ioc: per adapter object
 220  * @reply: reply message frame(lower 32bit addr)
 221  * @index: System request message index.
 222  */
 223 static void
 224 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,
 225                 u32 index)
 226 {
 227         /*
 228          * 256 is offset within sys register.
 229          * 256 offset MPI frame starts. Max MPI frame supported is 32.
 230          * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts
 231          */
 232         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 233         void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +
 234                         MPI_FRAME_START_OFFSET +
 235                         (cmd_credit * ioc->request_sz) + (index * sizeof(u32));
 236
 237         writel(reply, reply_free_iomem);
 238 }
 239
 240 /**
 241  * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames
 242  *                              to system/BAR0 region.
 243  *
 244  * @dst_iomem: Pointer to the destination location in BAR0 space.
 245  * @src: Pointer to the Source data.
 246  * @size: Size of data to be copied.
 247  */
 248 static void
 249 _base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size)
 250 {
 251         int i;
 252         u32 *src_virt_mem = (u32 *)src;
 253
 254         for (i = 0; i < size/4; i++)
 255                 writel((u32)src_virt_mem[i],
 256                                 (void __iomem *)dst_iomem + (i * 4));
 257 }
 258
 259 /**
 260  * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region
 261  *
 262  * @dst_iomem: Pointer to the destination location in BAR0 space.
 263  * @src: Pointer to the Source data.
 264  * @size: Size of data to be copied.
 265  */
 266 static void
 267 _base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size)
 268 {
 269         int i;
 270         u32 *src_virt_mem = (u32 *)(src);
 271
 272         for (i = 0; i < size/4; i++)
 273                 writel((u32)src_virt_mem[i],
 274                         (void __iomem *)dst_iomem + (i * 4));
 275 }
 276
 277 /**
 278  * _base_get_chain - Calculates and Returns virtual chain address
 279  *                       for the provided smid in BAR0 space.
 280  *
 281  * @ioc: per adapter object
 282  * @smid: system request message index
 283  * @sge_chain_count: Scatter gather chain count.
 284  *
 285  * Return: the chain address.
 286  */
 287 static inline void __iomem*
 288 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 289                 u8 sge_chain_count)
 290 {
 291         void __iomem *base_chain, *chain_virt;
 292         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 293
 294         base_chain  = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +
 295                 (cmd_credit * ioc->request_sz) +
 296                 REPLY_FREE_POOL_SIZE;
 297         chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *
 298                         ioc->request_sz) + (sge_chain_count * ioc->request_sz);
 299         return chain_virt;
 300 }
 301
 302 /**
 303  * _base_get_chain_phys - Calculates and Returns physical address
 304  *                      in BAR0 for scatter gather chains, for
 305  *                      the provided smid.
 306  *
 307  * @ioc: per adapter object
 308  * @smid: system request message index
 309  * @sge_chain_count: Scatter gather chain count.
 310  *
 311  * Return: Physical chain address.
 312  */
 313 static inline phys_addr_t
 314 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 315                 u8 sge_chain_count)
 316 {
 317         phys_addr_t base_chain_phys, chain_phys;
 318         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 319
 320         base_chain_phys  = ioc->chip_phys + MPI_FRAME_START_OFFSET +
 321                 (cmd_credit * ioc->request_sz) +
 322                 REPLY_FREE_POOL_SIZE;
 323         chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *
 324                         ioc->request_sz) + (sge_chain_count * ioc->request_sz);
 325         return chain_phys;
 326 }
 327
 328 /**
 329  * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host
 330  *                      buffer address for the provided smid.
 331  *                      (Each smid can have 64K starts from 17024)
 332  *
 333  * @ioc: per adapter object
 334  * @smid: system request message index
 335  *
 336  * Return: Pointer to buffer location in BAR0.
 337  */
 338
 339 static void __iomem *
 340 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 341 {
 342         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 343         // Added extra 1 to reach end of chain.
 344         void __iomem *chain_end = _base_get_chain(ioc,
 345                         cmd_credit + 1,
 346                         ioc->facts.MaxChainDepth);
 347         return chain_end + (smid * 64 * 1024);
 348 }
 349
 350 /**
 351  * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped
 352  *              Host buffer Physical address for the provided smid.
 353  *              (Each smid can have 64K starts from 17024)
 354  *
 355  * @ioc: per adapter object
 356  * @smid: system request message index
 357  *
 358  * Return: Pointer to buffer location in BAR0.
 359  */
 360 static phys_addr_t
 361 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 362 {
 363         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 364         phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,
 365                         cmd_credit + 1,
 366                         ioc->facts.MaxChainDepth);
 367         return chain_end_phys + (smid * 64 * 1024);
 368 }
 369
 370 /**
 371  * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain
 372  *                      lookup list and Provides chain_buffer
 373  *                      address for the matching dma address.
 374  *                      (Each smid can have 64K starts from 17024)
 375  *
 376  * @ioc: per adapter object
 377  * @chain_buffer_dma: Chain buffer dma address.
 378  *
 379  * Return: Pointer to chain buffer. Or Null on Failure.
 380  */
 381 static void *
 382 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
 383                 dma_addr_t chain_buffer_dma)
 384 {
 385         u16 index, j;
 386         struct chain_tracker *ct;
 387
 388         for (index = 0; index < ioc->scsiio_depth; index++) {
 389                 for (j = 0; j < ioc->chains_needed_per_io; j++) {
 390                         ct = &ioc->chain_lookup[index].chains_per_smid[j];
 391                         if (ct && ct->chain_buffer_dma == chain_buffer_dma)
 392                                 return ct->chain_buffer;
 393                 }
 394         }
 395         ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n");
 396         return NULL;
 397 }
 398
 399 /**
 400  * _clone_sg_entries -  MPI EP's scsiio and config requests
 401  *                      are handled here. Base function for
 402  *                      double buffering, before submitting
 403  *                      the requests.
 404  *
 405  * @ioc: per adapter object.
 406  * @mpi_request: mf request pointer.
 407  * @smid: system request message index.
 408  */
 409 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 410                 void *mpi_request, u16 smid)
 411 {
 412         Mpi2SGESimple32_t *sgel, *sgel_next;
 413         u32  sgl_flags, sge_chain_count = 0;
 414         bool is_write = false;
 415         u16 i = 0;
 416         void __iomem *buffer_iomem;
 417         phys_addr_t buffer_iomem_phys;
 418         void __iomem *buff_ptr;
 419         phys_addr_t buff_ptr_phys;
 420         void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO];
 421         void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO];
 422         phys_addr_t dst_addr_phys;
 423         MPI2RequestHeader_t *request_hdr;
 424         struct scsi_cmnd *scmd;
 425         struct scatterlist *sg_scmd = NULL;
 426         int is_scsiio_req = 0;
 427
 428         request_hdr = (MPI2RequestHeader_t *) mpi_request;
 429
 430         if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
 431                 Mpi25SCSIIORequest_t *scsiio_request =
 432                         (Mpi25SCSIIORequest_t *)mpi_request;
 433                 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL;
 434                 is_scsiio_req = 1;
 435         } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
 436                 Mpi2ConfigRequest_t  *config_req =
 437                         (Mpi2ConfigRequest_t *)mpi_request;
 438                 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE;
 439         } else
 440                 return;
 441
 442         /* From smid we can get scsi_cmd, once we have sg_scmd,
 443          * we just need to get sg_virt and sg_next to get virual
 444          * address associated with sgel->Address.
 445          */
 446
 447         if (is_scsiio_req) {
 448                 /* Get scsi_cmd using smid */
 449                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
 450                 if (scmd == NULL) {
 451                         ioc_err(ioc, "scmd is NULL\n");
 452                         return;
 453                 }
 454
 455                 /* Get sg_scmd from scmd provided */
 456                 sg_scmd = scsi_sglist(scmd);
 457         }
 458
 459         /*
 460          * 0 - 255      System register
 461          * 256 - 4352   MPI Frame. (This is based on maxCredit 32)
 462          * 4352 - 4864  Reply_free pool (512 byte is reserved
 463          *              considering maxCredit 32. Reply need extra
 464          *              room, for mCPU case kept four times of
 465          *              maxCredit).
 466          * 4864 - 17152 SGE chain element. (32cmd * 3 chain of
 467          *              128 byte size = 12288)
 468          * 17152 - x    Host buffer mapped with smid.
 469          *              (Each smid can have 64K Max IO.)
 470          * BAR0+Last 1K MSIX Addr and Data
 471          * Total size in use 2113664 bytes of 4MB BAR0
 472          */
 473
 474         buffer_iomem = _base_get_buffer_bar0(ioc, smid);
 475         buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);
 476
 477         buff_ptr = buffer_iomem;
 478         buff_ptr_phys = buffer_iomem_phys;
 479         WARN_ON(buff_ptr_phys > U32_MAX);
 480
 481         if (le32_to_cpu(sgel->FlagsLength) &
 482                         (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
 483                 is_write = true;
 484
 485         for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
 486
 487                 sgl_flags =
 488                     (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
 489
 490                 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
 491                 case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
 492                         /*
 493                          * Helper function which on passing
 494                          * chain_buffer_dma returns chain_buffer. Get
 495                          * the virtual address for sgel->Address
 496                          */
 497                         sgel_next =
 498                                 _base_get_chain_buffer_dma_to_chain_buffer(ioc,
 499                                                 le32_to_cpu(sgel->Address));
 500                         if (sgel_next == NULL)
 501                                 return;
 502                         /*
 503                          * This is coping 128 byte chain
 504                          * frame (not a host buffer)
 505                          */
 506                         dst_chain_addr[sge_chain_count] =
 507                                 _base_get_chain(ioc,
 508                                         smid, sge_chain_count);
 509                         src_chain_addr[sge_chain_count] =
 510                                                 (void *) sgel_next;
 511                         dst_addr_phys = _base_get_chain_phys(ioc,
 512                                                 smid, sge_chain_count);
 513                         WARN_ON(dst_addr_phys > U32_MAX);
 514                         sgel->Address =
 515                                 cpu_to_le32(lower_32_bits(dst_addr_phys));
 516                         sgel = sgel_next;
 517                         sge_chain_count++;
 518                         break;
 519                 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
 520                         if (is_write) {
 521                                 if (is_scsiio_req) {
 522                                         _base_clone_to_sys_mem(buff_ptr,
 523                                             sg_virt(sg_scmd),
 524                                             (le32_to_cpu(sgel->FlagsLength) &
 525                                             0x00ffffff));
 526                                         /*
 527                                          * FIXME: this relies on a a zero
 528                                          * PCI mem_offset.
 529                                          */
 530                                         sgel->Address =
 531                                             cpu_to_le32((u32)buff_ptr_phys);
 532                                 } else {
 533                                         _base_clone_to_sys_mem(buff_ptr,
 534                                             ioc->config_vaddr,
 535                                             (le32_to_cpu(sgel->FlagsLength) &
 536                                             0x00ffffff));
 537                                         sgel->Address =
 538                                             cpu_to_le32((u32)buff_ptr_phys);
 539                                 }
 540                         }
 541                         buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
 542                             0x00ffffff);
 543                         buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
 544                             0x00ffffff);
 545                         if ((le32_to_cpu(sgel->FlagsLength) &
 546                             (MPI2_SGE_FLAGS_END_OF_BUFFER
 547                                         << MPI2_SGE_FLAGS_SHIFT)))
 548                                 goto eob_clone_chain;
 549                         else {
 550                                 /*
 551                                  * Every single element in MPT will have
 552                                  * associated sg_next. Better to sanity that
 553                                  * sg_next is not NULL, but it will be a bug
 554                                  * if it is null.
 555                                  */
 556                                 if (is_scsiio_req) {
 557                                         sg_scmd = sg_next(sg_scmd);
 558                                         if (sg_scmd)
 559                                                 sgel++;
 560                                         else
 561                                                 goto eob_clone_chain;
 562                                 }
 563                         }
 564                         break;
 565                 }
 566         }
 567
 568 eob_clone_chain:
 569         for (i = 0; i < sge_chain_count; i++) {
 570                 if (is_scsiio_req)
 571                         _base_clone_to_sys_mem(dst_chain_addr[i],
 572                                 src_chain_addr[i], ioc->request_sz);
 573         }
 574 }
 575
 576 /**
 577  *  mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
 578  * @arg: input argument, used to derive ioc
 579  *
 580  * Return:
 581  * 0 if controller is removed from pci subsystem.
 582  * -1 for other case.
 583  */
 584 static int mpt3sas_remove_dead_ioc_func(void *arg)
 585 {
 586         struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
 587         struct pci_dev *pdev;
 588
 589         if (!ioc)
 590                 return -1;
 591
 592         pdev = ioc->pdev;
 593         if (!pdev)
 594                 return -1;
 595         pci_stop_and_remove_bus_device_locked(pdev);
 596         return 0;
 597 }
 598
 599 /**
 600  * _base_fault_reset_work - workq handling ioc fault conditions
 601  * @work: input argument, used to derive ioc
 602  *
 603  * Context: sleep.
 604  */
 605 static void
 606 _base_fault_reset_work(struct work_struct *work)
 607 {
 608         struct MPT3SAS_ADAPTER *ioc =
 609             container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
 610         unsigned long    flags;
 611         u32 doorbell;
 612         int rc;
 613         struct task_struct *p;
 614
 615
 616         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 617         if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) ||
 618                         ioc->pci_error_recovery)
 619                 goto rearm_timer;
 620         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 621
 622         doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 623         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
 624                 ioc_err(ioc, "SAS host is non-operational !!!!\n");
 625
 626                 /* It may be possible that EEH recovery can resolve some of
 627                  * pci bus failure issues rather removing the dead ioc function
 628                  * by considering controller is in a non-operational state. So
 629                  * here priority is given to the EEH recovery. If it doesn't
 630                  * not resolve this issue, mpt3sas driver will consider this
 631                  * controller to non-operational state and remove the dead ioc
 632                  * function.
 633                  */
 634                 if (ioc->non_operational_loop++ < 5) {
 635                         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,
 636                                                          flags);
 637                         goto rearm_timer;
 638                 }
 639
 640                 /*
 641                  * Call _scsih_flush_pending_cmds callback so that we flush all
 642                  * pending commands back to OS. This call is required to aovid
 643                  * deadlock at block layer. Dead IOC will fail to do diag reset,
 644                  * and this call is safe since dead ioc will never return any
 645                  * command back from HW.
 646                  */
 647                 ioc->schedule_dead_ioc_flush_running_cmds(ioc);
 648                 /*
 649                  * Set remove_host flag early since kernel thread will
 650                  * take some time to execute.
 651                  */
 652                 ioc->remove_host = 1;
 653                 /*Remove the Dead Host */
 654                 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
 655                     "%s_dead_ioc_%d", ioc->driver_name, ioc->id);
 656                 if (IS_ERR(p))
 657                         ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
 658                                 __func__);
 659                 else
 660                         ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
 661                                 __func__);
 662                 return; /* don't rearm timer */
 663         }
 664
 665         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) {
 666                 u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?
 667                     ioc->manu_pg11.CoreDumpTOSec :
 668                     MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS;
 669
 670                 timeout /= (FAULT_POLLING_INTERVAL/1000);
 671
 672                 if (ioc->ioc_coredump_loop == 0) {
 673                         mpt3sas_print_coredump_info(ioc,
 674                             doorbell & MPI2_DOORBELL_DATA_MASK);
 675                         /* do not accept any IOs and disable the interrupts */
 676                         spin_lock_irqsave(
 677                             &ioc->ioc_reset_in_progress_lock, flags);
 678                         ioc->shost_recovery = 1;
 679                         spin_unlock_irqrestore(
 680                             &ioc->ioc_reset_in_progress_lock, flags);
 681                         mpt3sas_base_mask_interrupts(ioc);
 682                         _base_clear_outstanding_commands(ioc);
 683                 }
 684
 685                 ioc_info(ioc, "%s: CoreDump loop %d.",
 686                     __func__, ioc->ioc_coredump_loop);
 687
 688                 /* Wait until CoreDump completes or times out */
 689                 if (ioc->ioc_coredump_loop++ < timeout) {
 690                         spin_lock_irqsave(
 691                             &ioc->ioc_reset_in_progress_lock, flags);
 692                         goto rearm_timer;
 693                 }
 694         }
 695
 696         if (ioc->ioc_coredump_loop) {
 697                 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_COREDUMP)
 698                         ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d",
 699                             __func__, ioc->ioc_coredump_loop);
 700                 else
 701                         ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d",
 702                             __func__, ioc->ioc_coredump_loop);
 703                 ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE;
 704         }
 705         ioc->non_operational_loop = 0;
 706         if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
 707                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 708                 ioc_warn(ioc, "%s: hard reset: %s\n",
 709                          __func__, rc == 0 ? "success" : "failed");
 710                 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 711                 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
 712                         mpt3sas_print_fault_code(ioc, doorbell &
 713                             MPI2_DOORBELL_DATA_MASK);
 714                 } else if ((doorbell & MPI2_IOC_STATE_MASK) ==
 715                     MPI2_IOC_STATE_COREDUMP)
 716                         mpt3sas_print_coredump_info(ioc, doorbell &
 717                             MPI2_DOORBELL_DATA_MASK);
 718                 if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
 719                     MPI2_IOC_STATE_OPERATIONAL)
 720                         return; /* don't rearm timer */
 721         }
 722         ioc->ioc_coredump_loop = 0;
 723
 724         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 725  rearm_timer:
 726         if (ioc->fault_reset_work_q)
 727                 queue_delayed_work(ioc->fault_reset_work_q,
 728                     &ioc->fault_reset_work,
 729                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 730         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 731 }
 732
 733 /**
 734  * mpt3sas_base_start_watchdog - start the fault_reset_work_q
 735  * @ioc: per adapter object
 736  *
 737  * Context: sleep.
 738  */
 739 void
 740 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
 741 {
 742         unsigned long    flags;
 743
 744         if (ioc->fault_reset_work_q)
 745                 return;
 746
 747         /* initialize fault polling */
 748
 749         INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
 750         snprintf(ioc->fault_reset_work_q_name,
 751             sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
 752             ioc->driver_name, ioc->id);
 753         ioc->fault_reset_work_q =
 754                 create_singlethread_workqueue(ioc->fault_reset_work_q_name);
 755         if (!ioc->fault_reset_work_q) {
 756                 ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__);
 757                 return;
 758         }
 759         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 760         if (ioc->fault_reset_work_q)
 761                 queue_delayed_work(ioc->fault_reset_work_q,
 762                     &ioc->fault_reset_work,
 763                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 764         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 765 }
 766
 767 /**
 768  * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
 769  * @ioc: per adapter object
 770  *
 771  * Context: sleep.
 772  */
 773 void
 774 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
 775 {
 776         unsigned long flags;
 777         struct workqueue_struct *wq;
 778
 779         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 780         wq = ioc->fault_reset_work_q;
 781         ioc->fault_reset_work_q = NULL;
 782         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 783         if (wq) {
 784                 if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
 785                         flush_workqueue(wq);
 786                 destroy_workqueue(wq);
 787         }
 788 }
 789
 790 /**
 791  * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
 792  * @ioc: per adapter object
 793  * @fault_code: fault code
 794  */
 795 void
 796 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
 797 {
 798         ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code);
 799 }
 800
 801 /**
 802  * mpt3sas_base_coredump_info - verbose translation of firmware CoreDump state
 803  * @ioc: per adapter object
 804  * @fault_code: fault code
 805  *
 806  * Return nothing.
 807  */
 808 void
 809 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)
 810 {
 811         ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code);
 812 }
 813
 814 /**
 815  * mpt3sas_base_wait_for_coredump_completion - Wait until coredump
 816  * completes or times out
 817  * @ioc: per adapter object
 818  * @caller: caller function name
 819  *
 820  * Returns 0 for success, non-zero for failure.
 821  */
 822 int
 823 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc,
 824                 const char *caller)
 825 {
 826         u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?
 827                         ioc->manu_pg11.CoreDumpTOSec :
 828                         MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS;
 829
 830         int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT,
 831                                         timeout);
 832
 833         if (ioc_state)
 834                 ioc_err(ioc,
 835                     "%s: CoreDump timed out. (ioc_state=0x%x)\n",
 836                     caller, ioc_state);
 837         else
 838                 ioc_info(ioc,
 839                     "%s: CoreDump completed. (ioc_state=0x%x)\n",
 840                     caller, ioc_state);
 841
 842         return ioc_state;
 843 }
 844
 845 /**
 846  * mpt3sas_halt_firmware - halt's mpt controller firmware
 847  * @ioc: per adapter object
 848  *
 849  * For debugging timeout related issues.  Writing 0xCOFFEE00
 850  * to the doorbell register will halt controller firmware. With
 851  * the purpose to stop both driver and firmware, the enduser can
 852  * obtain a ring buffer from controller UART.
 853  */
 854 void
 855 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
 856 {
 857         u32 doorbell;
 858
 859         if (!ioc->fwfault_debug)
 860                 return;
 861
 862         dump_stack();
 863
 864         doorbell = ioc->base_readl(&ioc->chip->Doorbell);
 865         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
 866                 mpt3sas_print_fault_code(ioc, doorbell &
 867                     MPI2_DOORBELL_DATA_MASK);
 868         } else if ((doorbell & MPI2_IOC_STATE_MASK) ==
 869             MPI2_IOC_STATE_COREDUMP) {
 870                 mpt3sas_print_coredump_info(ioc, doorbell &
 871                     MPI2_DOORBELL_DATA_MASK);
 872         } else {
 873                 writel(0xC0FFEE00, &ioc->chip->Doorbell);
 874                 ioc_err(ioc, "Firmware is halted due to command timeout\n");
 875         }
 876
 877         if (ioc->fwfault_debug == 2)
 878                 for (;;)
 879                         ;
 880         else
 881                 panic("panic in %s\n", __func__);
 882 }
 883
 884 /**
 885  * _base_sas_ioc_info - verbose translation of the ioc status
 886  * @ioc: per adapter object
 887  * @mpi_reply: reply mf payload returned from firmware
 888  * @request_hdr: request mf
 889  */
 890 static void
 891 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
 892         MPI2RequestHeader_t *request_hdr)
 893 {
 894         u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
 895             MPI2_IOCSTATUS_MASK;
 896         char *desc = NULL;
 897         u16 frame_sz;
 898         char *func_str = NULL;
 899
 900         /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
 901         if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
 902             request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
 903             request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
 904                 return;
 905
 906         if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 907                 return;
 908
 909         switch (ioc_status) {
 910
 911 /****************************************************************************
 912 *  Common IOCStatus values for all replies
 913 ****************************************************************************/
 914
 915         case MPI2_IOCSTATUS_INVALID_FUNCTION:
 916                 desc = "invalid function";
 917                 break;
 918         case MPI2_IOCSTATUS_BUSY:
 919                 desc = "busy";
 920                 break;
 921         case MPI2_IOCSTATUS_INVALID_SGL:
 922                 desc = "invalid sgl";
 923                 break;
 924         case MPI2_IOCSTATUS_INTERNAL_ERROR:
 925                 desc = "internal error";
 926                 break;
 927         case MPI2_IOCSTATUS_INVALID_VPID:
 928                 desc = "invalid vpid";
 929                 break;
 930         case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
 931                 desc = "insufficient resources";
 932                 break;
 933         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
 934                 desc = "insufficient power";
 935                 break;
 936         case MPI2_IOCSTATUS_INVALID_FIELD:
 937                 desc = "invalid field";
 938                 break;
 939         case MPI2_IOCSTATUS_INVALID_STATE:
 940                 desc = "invalid state";
 941                 break;
 942         case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
 943                 desc = "op state not supported";
 944                 break;
 945
 946 /****************************************************************************
 947 *  Config IOCStatus values
 948 ****************************************************************************/
 949
 950         case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
 951                 desc = "config invalid action";
 952                 break;
 953         case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
 954                 desc = "config invalid type";
 955                 break;
 956         case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
 957                 desc = "config invalid page";
 958                 break;
 959         case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
 960                 desc = "config invalid data";
 961                 break;
 962         case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
 963                 desc = "config no defaults";
 964                 break;
 965         case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
 966                 desc = "config cant commit";
 967                 break;
 968
 969 /****************************************************************************
 970 *  SCSI IO Reply
 971 ****************************************************************************/
 972
 973         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
 974         case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
 975         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
 976         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
 977         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
 978         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
 979         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
 980         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
 981         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
 982         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
 983         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
 984         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
 985                 break;
 986
 987 /****************************************************************************
 988 *  For use by SCSI Initiator and SCSI Target end-to-end data protection
 989 ****************************************************************************/
 990
 991         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
 992                 desc = "eedp guard error";
 993                 break;
 994         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
 995                 desc = "eedp ref tag error";
 996                 break;
 997         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
 998                 desc = "eedp app tag error";
 999                 break;
1000
1001 /****************************************************************************
1002 *  SCSI Target values
1003 ****************************************************************************/
1004
1005         case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
1006                 desc = "target invalid io index";
1007                 break;
1008         case MPI2_IOCSTATUS_TARGET_ABORTED:
1009                 desc = "target aborted";
1010                 break;
1011         case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
1012                 desc = "target no conn retryable";
1013                 break;
1014         case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
1015                 desc = "target no connection";
1016                 break;
1017         case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
1018                 desc = "target xfer count mismatch";
1019                 break;
1020         case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
1021                 desc = "target data offset error";
1022                 break;
1023         case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
1024                 desc = "target too much write data";
1025                 break;
1026         case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
1027                 desc = "target iu too short";
1028                 break;
1029         case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
1030                 desc = "target ack nak timeout";
1031                 break;
1032         case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
1033                 desc = "target nak received";
1034                 break;
1035
1036 /****************************************************************************
1037 *  Serial Attached SCSI values
1038 ****************************************************************************/
1039
1040         case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
1041                 desc = "smp request failed";
1042                 break;
1043         case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
1044                 desc = "smp data overrun";
1045                 break;
1046
1047 /****************************************************************************
1048 *  Diagnostic Buffer Post / Diagnostic Release values
1049 ****************************************************************************/
1050
1051         case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
1052                 desc = "diagnostic released";
1053                 break;
1054         default:
1055                 break;
1056         }
1057
1058         if (!desc)
1059                 return;
1060
1061         switch (request_hdr->Function) {
1062         case MPI2_FUNCTION_CONFIG:
1063                 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
1064                 func_str = "config_page";
1065                 break;
1066         case MPI2_FUNCTION_SCSI_TASK_MGMT:
1067                 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
1068                 func_str = "task_mgmt";
1069                 break;
1070         case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
1071                 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
1072                 func_str = "sas_iounit_ctl";
1073                 break;
1074         case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
1075                 frame_sz = sizeof(Mpi2SepRequest_t);
1076                 func_str = "enclosure";
1077                 break;
1078         case MPI2_FUNCTION_IOC_INIT:
1079                 frame_sz = sizeof(Mpi2IOCInitRequest_t);
1080                 func_str = "ioc_init";
1081                 break;
1082         case MPI2_FUNCTION_PORT_ENABLE:
1083                 frame_sz = sizeof(Mpi2PortEnableRequest_t);
1084                 func_str = "port_enable";
1085                 break;
1086         case MPI2_FUNCTION_SMP_PASSTHROUGH:
1087                 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
1088                 func_str = "smp_passthru";
1089                 break;
1090         case MPI2_FUNCTION_NVME_ENCAPSULATED:
1091                 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) +
1092                     ioc->sge_size;
1093                 func_str = "nvme_encapsulated";
1094                 break;
1095         default:
1096                 frame_sz = 32;
1097                 func_str = "unknown";
1098                 break;
1099         }
1100
1101         ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
1102                  desc, ioc_status, request_hdr, func_str);
1103
1104         _debug_dump_mf(request_hdr, frame_sz/4);
1105 }
1106
1107 /**
1108  * _base_display_event_data - verbose translation of firmware asyn events
1109  * @ioc: per adapter object
1110  * @mpi_reply: reply mf payload returned from firmware
1111  */
1112 static void
1113 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
1114         Mpi2EventNotificationReply_t *mpi_reply)
1115 {
1116         char *desc = NULL;
1117         u16 event;
1118
1119         if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
1120                 return;
1121
1122         event = le16_to_cpu(mpi_reply->Event);
1123
1124         switch (event) {
1125         case MPI2_EVENT_LOG_DATA:
1126                 desc = "Log Data";
1127                 break;
1128         case MPI2_EVENT_STATE_CHANGE:
1129                 desc = "Status Change";
1130                 break;
1131         case MPI2_EVENT_HARD_RESET_RECEIVED:
1132                 desc = "Hard Reset Received";
1133                 break;
1134         case MPI2_EVENT_EVENT_CHANGE:
1135                 desc = "Event Change";
1136                 break;
1137         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
1138                 desc = "Device Status Change";
1139                 break;
1140         case MPI2_EVENT_IR_OPERATION_STATUS:
1141                 if (!ioc->hide_ir_msg)
1142                         desc = "IR Operation Status";
1143                 break;
1144         case MPI2_EVENT_SAS_DISCOVERY:
1145         {
1146                 Mpi2EventDataSasDiscovery_t *event_data =
1147                     (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
1148                 ioc_info(ioc, "Discovery: (%s)",
1149                          event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ?
1150                          "start" : "stop");
1151                 if (event_data->DiscoveryStatus)
1152                         pr_cont(" discovery_status(0x%08x)",
1153                             le32_to_cpu(event_data->DiscoveryStatus));
1154                 pr_cont("\n");
1155                 return;
1156         }
1157         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
1158                 desc = "SAS Broadcast Primitive";
1159                 break;
1160         case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
1161                 desc = "SAS Init Device Status Change";
1162                 break;
1163         case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
1164                 desc = "SAS Init Table Overflow";
1165                 break;
1166         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1167                 desc = "SAS Topology Change List";
1168                 break;
1169         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
1170                 desc = "SAS Enclosure Device Status Change";
1171                 break;
1172         case MPI2_EVENT_IR_VOLUME:
1173                 if (!ioc->hide_ir_msg)
1174                         desc = "IR Volume";
1175                 break;
1176         case MPI2_EVENT_IR_PHYSICAL_DISK:
1177                 if (!ioc->hide_ir_msg)
1178                         desc = "IR Physical Disk";
1179                 break;
1180         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
1181                 if (!ioc->hide_ir_msg)
1182                         desc = "IR Configuration Change List";
1183                 break;
1184         case MPI2_EVENT_LOG_ENTRY_ADDED:
1185                 if (!ioc->hide_ir_msg)
1186                         desc = "Log Entry Added";
1187                 break;
1188         case MPI2_EVENT_TEMP_THRESHOLD:
1189                 desc = "Temperature Threshold";
1190                 break;
1191         case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
1192                 desc = "Cable Event";
1193                 break;
1194         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
1195                 desc = "SAS Device Discovery Error";
1196                 break;
1197         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
1198                 desc = "PCIE Device Status Change";
1199                 break;
1200         case MPI2_EVENT_PCIE_ENUMERATION:
1201         {
1202                 Mpi26EventDataPCIeEnumeration_t *event_data =
1203                         (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData;
1204                 ioc_info(ioc, "PCIE Enumeration: (%s)",
1205                          event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED ?
1206                          "start" : "stop");
1207                 if (event_data->EnumerationStatus)
1208                         pr_cont("enumeration_status(0x%08x)",
1209                                 le32_to_cpu(event_data->EnumerationStatus));
1210                 pr_cont("\n");
1211                 return;
1212         }
1213         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1214                 desc = "PCIE Topology Change List";
1215                 break;
1216         }
1217
1218         if (!desc)
1219                 return;
1220
1221         ioc_info(ioc, "%s\n", desc);
1222 }
1223
1224 /**
1225  * _base_sas_log_info - verbose translation of firmware log info
1226  * @ioc: per adapter object
1227  * @log_info: log info
1228  */
1229 static void
1230 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
1231 {
1232         union loginfo_type {
1233                 u32     loginfo;
1234                 struct {
1235                         u32     subcode:16;
1236                         u32     code:8;
1237                         u32     originator:4;
1238                         u32     bus_type:4;
1239                 } dw;
1240         };
1241         union loginfo_type sas_loginfo;
1242         char *originator_str = NULL;
1243
1244         sas_loginfo.loginfo = log_info;
1245         if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1246                 return;
1247
1248         /* each nexus loss loginfo */
1249         if (log_info == 0x31170000)
1250                 return;
1251
1252         /* eat the loginfos associated with task aborts */
1253         if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
1254             0x31140000 || log_info == 0x31130000))
1255                 return;
1256
1257         switch (sas_loginfo.dw.originator) {
1258         case 0:
1259                 originator_str = "IOP";
1260                 break;
1261         case 1:
1262                 originator_str = "PL";
1263                 break;
1264         case 2:
1265                 if (!ioc->hide_ir_msg)
1266                         originator_str = "IR";
1267                 else
1268                         originator_str = "WarpDrive";
1269                 break;
1270         }
1271
1272         ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
1273                  log_info,
1274                  originator_str, sas_loginfo.dw.code, sas_loginfo.dw.subcode);
1275 }
1276
1277 /**
1278  * _base_display_reply_info -
1279  * @ioc: per adapter object
1280  * @smid: system request message index
1281  * @msix_index: MSIX table index supplied by the OS
1282  * @reply: reply message frame(lower 32bit addr)
1283  */
1284 static void
1285 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1286         u32 reply)
1287 {
1288         MPI2DefaultReply_t *mpi_reply;
1289         u16 ioc_status;
1290         u32 loginfo = 0;
1291
1292         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1293         if (unlikely(!mpi_reply)) {
1294                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
1295                         __FILE__, __LINE__, __func__);
1296                 return;
1297         }
1298         ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
1299
1300         if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
1301             (ioc->logging_level & MPT_DEBUG_REPLY)) {
1302                 _base_sas_ioc_info(ioc , mpi_reply,
1303                    mpt3sas_base_get_msg_frame(ioc, smid));
1304         }
1305
1306         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
1307                 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
1308                 _base_sas_log_info(ioc, loginfo);
1309         }
1310
1311         if (ioc_status || loginfo) {
1312                 ioc_status &= MPI2_IOCSTATUS_MASK;
1313                 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
1314         }
1315 }
1316
1317 /**
1318  * mpt3sas_base_done - base internal command completion routine
1319  * @ioc: per adapter object
1320  * @smid: system request message index
1321  * @msix_index: MSIX table index supplied by the OS
1322  * @reply: reply message frame(lower 32bit addr)
1323  *
1324  * Return:
1325  * 1 meaning mf should be freed from _base_interrupt
1326  * 0 means the mf is freed from this function.
1327  */
1328 u8
1329 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1330         u32 reply)
1331 {
1332         MPI2DefaultReply_t *mpi_reply;
1333
1334         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1335         if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
1336                 return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
1337
1338         if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
1339                 return 1;
1340
1341         ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
1342         if (mpi_reply) {
1343                 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
1344                 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
1345         }
1346         ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
1347
1348         complete(&ioc->base_cmds.done);
1349         return 1;
1350 }
1351
1352 /**
1353  * _base_async_event - main callback handler for firmware asyn events
1354  * @ioc: per adapter object
1355  * @msix_index: MSIX table index supplied by the OS
1356  * @reply: reply message frame(lower 32bit addr)
1357  *
1358  * Return:
1359  * 1 meaning mf should be freed from _base_interrupt
1360  * 0 means the mf is freed from this function.
1361  */
1362 static u8
1363 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
1364 {
1365         Mpi2EventNotificationReply_t *mpi_reply;
1366         Mpi2EventAckRequest_t *ack_request;
1367         u16 smid;
1368         struct _event_ack_list *delayed_event_ack;
1369
1370         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1371         if (!mpi_reply)
1372                 return 1;
1373         if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
1374                 return 1;
1375
1376         _base_display_event_data(ioc, mpi_reply);
1377
1378         if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
1379                 goto out;
1380         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
1381         if (!smid) {
1382                 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack),
1383                                         GFP_ATOMIC);
1384                 if (!delayed_event_ack)
1385                         goto out;
1386                 INIT_LIST_HEAD(&delayed_event_ack->list);
1387                 delayed_event_ack->Event = mpi_reply->Event;
1388                 delayed_event_ack->EventContext = mpi_reply->EventContext;
1389                 list_add_tail(&delayed_event_ack->list,
1390                                 &ioc->delayed_event_ack_list);
1391                 dewtprintk(ioc,
1392                            ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n",
1393                                     le16_to_cpu(mpi_reply->Event)));
1394                 goto out;
1395         }
1396
1397         ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
1398         memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
1399         ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
1400         ack_request->Event = mpi_reply->Event;
1401         ack_request->EventContext = mpi_reply->EventContext;
1402         ack_request->VF_ID = 0;  /* TODO */
1403         ack_request->VP_ID = 0;
1404         ioc->put_smid_default(ioc, smid);
1405
1406  out:
1407
1408         /* scsih callback handler */
1409         mpt3sas_scsih_event_callback(ioc, msix_index, reply);
1410
1411         /* ctl callback handler */
1412         mpt3sas_ctl_event_callback(ioc, msix_index, reply);
1413
1414         return 1;
1415 }
1416
1417 static struct scsiio_tracker *
1418 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1419 {
1420         struct scsi_cmnd *cmd;
1421
1422         if (WARN_ON(!smid) ||
1423             WARN_ON(smid >= ioc->hi_priority_smid))
1424                 return NULL;
1425
1426         cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1427         if (cmd)
1428                 return scsi_cmd_priv(cmd);
1429
1430         return NULL;
1431 }
1432
1433 /**
1434  * _base_get_cb_idx - obtain the callback index
1435  * @ioc: per adapter object
1436  * @smid: system request message index
1437  *
1438  * Return: callback index.
1439  */
1440 static u8
1441 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1442 {
1443         int i;
1444         u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
1445         u8 cb_idx = 0xFF;
1446
1447         if (smid < ioc->hi_priority_smid) {
1448                 struct scsiio_tracker *st;
1449
1450                 if (smid < ctl_smid) {
1451                         st = _get_st_from_smid(ioc, smid);
1452                         if (st)
1453                                 cb_idx = st->cb_idx;
1454                 } else if (smid == ctl_smid)
1455                         cb_idx = ioc->ctl_cb_idx;
1456         } else if (smid < ioc->internal_smid) {
1457                 i = smid - ioc->hi_priority_smid;
1458                 cb_idx = ioc->hpr_lookup[i].cb_idx;
1459         } else if (smid <= ioc->hba_queue_depth) {
1460                 i = smid - ioc->internal_smid;
1461                 cb_idx = ioc->internal_lookup[i].cb_idx;
1462         }
1463         return cb_idx;
1464 }
1465
1466 /**
1467  * mpt3sas_base_mask_interrupts - disable interrupts
1468  * @ioc: per adapter object
1469  *
1470  * Disabling ResetIRQ, Reply and Doorbell Interrupts
1471  */
1472 void
1473 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1474 {
1475         u32 him_register;
1476
1477         ioc->mask_interrupts = 1;
1478         him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);
1479         him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
1480         writel(him_register, &ioc->chip->HostInterruptMask);
1481         ioc->base_readl(&ioc->chip->HostInterruptMask);
1482 }
1483
1484 /**
1485  * mpt3sas_base_unmask_interrupts - enable interrupts
1486  * @ioc: per adapter object
1487  *
1488  * Enabling only Reply Interrupts
1489  */
1490 void
1491 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1492 {
1493         u32 him_register;
1494
1495         him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);
1496         him_register &= ~MPI2_HIM_RIM;
1497         writel(him_register, &ioc->chip->HostInterruptMask);
1498         ioc->mask_interrupts = 0;
1499 }
1500
1501 union reply_descriptor {
1502         u64 word;
1503         struct {
1504                 u32 low;
1505                 u32 high;
1506         } u;
1507 };
1508
1509 static u32 base_mod64(u64 dividend, u32 divisor)
1510 {
1511         u32 remainder;
1512
1513         if (!divisor)
1514                 pr_err("mpt3sas: DIVISOR is zero, in div fn\n");
1515         remainder = do_div(dividend, divisor);
1516         return remainder;
1517 }
1518
1519 /**
1520  * _base_process_reply_queue - Process reply descriptors from reply
1521  *              descriptor post queue.
1522  * @reply_q: per IRQ's reply queue object.
1523  *
1524  * Return: number of reply descriptors processed from reply
1525  *              descriptor queue.
1526  */
1527 static int
1528 _base_process_reply_queue(struct adapter_reply_queue *reply_q)
1529 {
1530         union reply_descriptor rd;
1531         u64 completed_cmds;
1532         u8 request_descript_type;
1533         u16 smid;
1534         u8 cb_idx;
1535         u32 reply;
1536         u8 msix_index = reply_q->msix_index;
1537         struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1538         Mpi2ReplyDescriptorsUnion_t *rpf;
1539         u8 rc;
1540
1541         completed_cmds = 0;
1542         if (!atomic_add_unless(&reply_q->busy, 1, 1))
1543                 return completed_cmds;
1544
1545         rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
1546         request_descript_type = rpf->Default.ReplyFlags
1547              & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1548         if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
1549                 atomic_dec(&reply_q->busy);
1550                 return completed_cmds;
1551         }
1552
1553         cb_idx = 0xFF;
1554         do {
1555                 rd.word = le64_to_cpu(rpf->Words);
1556                 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
1557                         goto out;
1558                 reply = 0;
1559                 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
1560                 if (request_descript_type ==
1561                     MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
1562                     request_descript_type ==
1563                     MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
1564                     request_descript_type ==
1565                     MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
1566                         cb_idx = _base_get_cb_idx(ioc, smid);
1567                         if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1568                             (likely(mpt_callbacks[cb_idx] != NULL))) {
1569                                 rc = mpt_callbacks[cb_idx](ioc, smid,
1570                                     msix_index, 0);
1571                                 if (rc)
1572                                         mpt3sas_base_free_smid(ioc, smid);
1573                         }
1574                 } else if (request_descript_type ==
1575                     MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
1576                         reply = le32_to_cpu(
1577                             rpf->AddressReply.ReplyFrameAddress);
1578                         if (reply > ioc->reply_dma_max_address ||
1579                             reply < ioc->reply_dma_min_address)
1580                                 reply = 0;
1581                         if (smid) {
1582                                 cb_idx = _base_get_cb_idx(ioc, smid);
1583                                 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1584                                     (likely(mpt_callbacks[cb_idx] != NULL))) {
1585                                         rc = mpt_callbacks[cb_idx](ioc, smid,
1586                                             msix_index, reply);
1587                                         if (reply)
1588                                                 _base_display_reply_info(ioc,
1589                                                     smid, msix_index, reply);
1590                                         if (rc)
1591                                                 mpt3sas_base_free_smid(ioc,
1592                                                     smid);
1593                                 }
1594                         } else {
1595                                 _base_async_event(ioc, msix_index, reply);
1596                         }
1597
1598                         /* reply free queue handling */
1599                         if (reply) {
1600                                 ioc->reply_free_host_index =
1601                                     (ioc->reply_free_host_index ==
1602                                     (ioc->reply_free_queue_depth - 1)) ?
1603                                     0 : ioc->reply_free_host_index + 1;
1604                                 ioc->reply_free[ioc->reply_free_host_index] =
1605                                     cpu_to_le32(reply);
1606                                 if (ioc->is_mcpu_endpoint)
1607                                         _base_clone_reply_to_sys_mem(ioc,
1608                                                 reply,
1609                                                 ioc->reply_free_host_index);
1610                                 writel(ioc->reply_free_host_index,
1611                                     &ioc->chip->ReplyFreeHostIndex);
1612                         }
1613                 }
1614
1615                 rpf->Words = cpu_to_le64(ULLONG_MAX);
1616                 reply_q->reply_post_host_index =
1617                     (reply_q->reply_post_host_index ==
1618                     (ioc->reply_post_queue_depth - 1)) ? 0 :
1619                     reply_q->reply_post_host_index + 1;
1620                 request_descript_type =
1621                     reply_q->reply_post_free[reply_q->reply_post_host_index].
1622                     Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1623                 completed_cmds++;
1624                 /* Update the reply post host index after continuously
1625                  * processing the threshold number of Reply Descriptors.
1626                  * So that FW can find enough entries to post the Reply
1627                  * Descriptors in the reply descriptor post queue.
1628                  */
1629                 if (completed_cmds >= ioc->thresh_hold) {
1630                         if (ioc->combined_reply_queue) {
1631                                 writel(reply_q->reply_post_host_index |
1632                                                 ((msix_index  & 7) <<
1633                                                  MPI2_RPHI_MSIX_INDEX_SHIFT),
1634                                     ioc->replyPostRegisterIndex[msix_index/8]);
1635                         } else {
1636                                 writel(reply_q->reply_post_host_index |
1637                                                 (msix_index <<
1638                                                  MPI2_RPHI_MSIX_INDEX_SHIFT),
1639                                                 &ioc->chip->ReplyPostHostIndex);
1640                         }
1641                         if (!reply_q->irq_poll_scheduled) {
1642                                 reply_q->irq_poll_scheduled = true;
1643                                 irq_poll_sched(&reply_q->irqpoll);
1644                         }
1645                         atomic_dec(&reply_q->busy);
1646                         return completed_cmds;
1647                 }
1648                 if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1649                         goto out;
1650                 if (!reply_q->reply_post_host_index)
1651                         rpf = reply_q->reply_post_free;
1652                 else
1653                         rpf++;
1654         } while (1);
1655
1656  out:
1657
1658         if (!completed_cmds) {
1659                 atomic_dec(&reply_q->busy);
1660                 return completed_cmds;
1661         }
1662
1663         if (ioc->is_warpdrive) {
1664                 writel(reply_q->reply_post_host_index,
1665                 ioc->reply_post_host_index[msix_index]);
1666                 atomic_dec(&reply_q->busy);
1667                 return completed_cmds;
1668         }
1669
1670         /* Update Reply Post Host Index.
1671          * For those HBA's which support combined reply queue feature
1672          * 1. Get the correct Supplemental Reply Post Host Index Register.
1673          *    i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1674          *    Index Register address bank i.e replyPostRegisterIndex[],
1675          * 2. Then update this register with new reply host index value
1676          *    in ReplyPostIndex field and the MSIxIndex field with
1677          *    msix_index value reduced to a value between 0 and 7,
1678          *    using a modulo 8 operation. Since each Supplemental Reply Post
1679          *    Host Index Register supports 8 MSI-X vectors.
1680          *
1681          * For other HBA's just update the Reply Post Host Index register with
1682          * new reply host index value in ReplyPostIndex Field and msix_index
1683          * value in MSIxIndex field.
1684          */
1685         if (ioc->combined_reply_queue)
1686                 writel(reply_q->reply_post_host_index | ((msix_index  & 7) <<
1687                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1688                         ioc->replyPostRegisterIndex[msix_index/8]);
1689         else
1690                 writel(reply_q->reply_post_host_index | (msix_index <<
1691                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1692                         &ioc->chip->ReplyPostHostIndex);
1693         atomic_dec(&reply_q->busy);
1694         return completed_cmds;
1695 }
1696
1697 /**
1698  * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
1699  * @irq: irq number (not used)
1700  * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
1701  *
1702  * Return: IRQ_HANDLED if processed, else IRQ_NONE.
1703  */
1704 static irqreturn_t
1705 _base_interrupt(int irq, void *bus_id)
1706 {
1707         struct adapter_reply_queue *reply_q = bus_id;
1708         struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1709
1710         if (ioc->mask_interrupts)
1711                 return IRQ_NONE;
1712         if (reply_q->irq_poll_scheduled)
1713                 return IRQ_HANDLED;
1714         return ((_base_process_reply_queue(reply_q) > 0) ?
1715                         IRQ_HANDLED : IRQ_NONE);
1716 }
1717
1718 /**
1719  * _base_irqpoll - IRQ poll callback handler
1720  * @irqpoll: irq_poll object
1721  * @budget: irq poll weight
1722  *
1723  * returns number of reply descriptors processed
1724  */
1725 static int
1726 _base_irqpoll(struct irq_poll *irqpoll, int budget)
1727 {
1728         struct adapter_reply_queue *reply_q;
1729         int num_entries = 0;
1730
1731         reply_q = container_of(irqpoll, struct adapter_reply_queue,
1732                         irqpoll);
1733         if (reply_q->irq_line_enable) {
1734                 disable_irq_nosync(reply_q->os_irq);
1735                 reply_q->irq_line_enable = false;
1736         }
1737         num_entries = _base_process_reply_queue(reply_q);
1738         if (num_entries < budget) {
1739                 irq_poll_complete(irqpoll);
1740                 reply_q->irq_poll_scheduled = false;
1741                 reply_q->irq_line_enable = true;
1742                 enable_irq(reply_q->os_irq);
1743                 /*
1744                  * Go for one more round of processing the
1745                  * reply descriptor post queue incase if HBA
1746                  * Firmware has posted some reply descriptors
1747                  * while reenabling the IRQ.
1748                  */
1749                 _base_process_reply_queue(reply_q);
1750         }
1751
1752         return num_entries;
1753 }
1754
1755 /**
1756  * _base_init_irqpolls - initliaze IRQ polls
1757  * @ioc: per adapter object
1758  *
1759  * returns nothing
1760  */
1761 static void
1762 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc)
1763 {
1764         struct adapter_reply_queue *reply_q, *next;
1765
1766         if (list_empty(&ioc->reply_queue_list))
1767                 return;
1768
1769         list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
1770                 irq_poll_init(&reply_q->irqpoll,
1771                         ioc->hba_queue_depth/4, _base_irqpoll);
1772                 reply_q->irq_poll_scheduled = false;
1773                 reply_q->irq_line_enable = true;
1774                 reply_q->os_irq = pci_irq_vector(ioc->pdev,
1775                     reply_q->msix_index);
1776         }
1777 }
1778
1779 /**
1780  * _base_is_controller_msix_enabled - is controller support muli-reply queues
1781  * @ioc: per adapter object
1782  *
1783  * Return: Whether or not MSI/X is enabled.
1784  */
1785 static inline int
1786 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1787 {
1788         return (ioc->facts.IOCCapabilities &
1789             MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1790 }
1791
1792 /**
1793  * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts
1794  * @ioc: per adapter object
1795  * @poll: poll over reply descriptor pools incase interrupt for
1796  *              timed-out SCSI command got delayed
1797  * Context: non ISR conext
1798  *
1799  * Called when a Task Management request has completed.
1800  */
1801 void
1802 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll)
1803 {
1804         struct adapter_reply_queue *reply_q;
1805
1806         /* If MSIX capability is turned off
1807          * then multi-queues are not enabled
1808          */
1809         if (!_base_is_controller_msix_enabled(ioc))
1810                 return;
1811
1812         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
1813                 if (ioc->shost_recovery || ioc->remove_host ||
1814                                 ioc->pci_error_recovery)
1815                         return;
1816                 /* TMs are on msix_index == 0 */
1817                 if (reply_q->msix_index == 0)
1818                         continue;
1819                 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
1820                 if (reply_q->irq_poll_scheduled) {
1821                         /* Calling irq_poll_disable will wait for any pending
1822                          * callbacks to have completed.
1823                          */
1824                         irq_poll_disable(&reply_q->irqpoll);
1825                         irq_poll_enable(&reply_q->irqpoll);
1826                         /* check how the scheduled poll has ended,
1827                          * clean up only if necessary
1828                          */
1829                         if (reply_q->irq_poll_scheduled) {
1830                                 reply_q->irq_poll_scheduled = false;
1831                                 reply_q->irq_line_enable = true;
1832                                 enable_irq(reply_q->os_irq);
1833                         }
1834                 }
1835
1836                 if (poll)
1837                         _base_process_reply_queue(reply_q);
1838         }
1839 }
1840
1841 /**
1842  * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1843  * @cb_idx: callback index
1844  */
1845 void
1846 mpt3sas_base_release_callback_handler(u8 cb_idx)
1847 {
1848         mpt_callbacks[cb_idx] = NULL;
1849 }
1850
1851 /**
1852  * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1853  * @cb_func: callback function
1854  *
1855  * Return: Index of @cb_func.
1856  */
1857 u8
1858 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1859 {
1860         u8 cb_idx;
1861
1862         for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1863                 if (mpt_callbacks[cb_idx] == NULL)
1864                         break;
1865
1866         mpt_callbacks[cb_idx] = cb_func;
1867         return cb_idx;
1868 }
1869
1870 /**
1871  * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1872  */
1873 void
1874 mpt3sas_base_initialize_callback_handler(void)
1875 {
1876         u8 cb_idx;
1877
1878         for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1879                 mpt3sas_base_release_callback_handler(cb_idx);
1880 }
1881
1882
1883 /**
1884  * _base_build_zero_len_sge - build zero length sg entry
1885  * @ioc: per adapter object
1886  * @paddr: virtual address for SGE
1887  *
1888  * Create a zero length scatter gather entry to insure the IOCs hardware has
1889  * something to use if the target device goes brain dead and tries
1890  * to send data even when none is asked for.
1891  */
1892 static void
1893 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1894 {
1895         u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1896             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1897             MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1898             MPI2_SGE_FLAGS_SHIFT);
1899         ioc->base_add_sg_single(paddr, flags_length, -1);
1900 }
1901
1902 /**
1903  * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1904  * @paddr: virtual address for SGE
1905  * @flags_length: SGE flags and data transfer length
1906  * @dma_addr: Physical address
1907  */
1908 static void
1909 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1910 {
1911         Mpi2SGESimple32_t *sgel = paddr;
1912
1913         flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1914             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1915         sgel->FlagsLength = cpu_to_le32(flags_length);
1916         sgel->Address = cpu_to_le32(dma_addr);
1917 }
1918
1919
1920 /**
1921  * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
1922  * @paddr: virtual address for SGE
1923  * @flags_length: SGE flags and data transfer length
1924  * @dma_addr: Physical address
1925  */
1926 static void
1927 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1928 {
1929         Mpi2SGESimple64_t *sgel = paddr;
1930
1931         flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
1932             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1933         sgel->FlagsLength = cpu_to_le32(flags_length);
1934         sgel->Address = cpu_to_le64(dma_addr);
1935 }
1936
1937 /**
1938  * _base_get_chain_buffer_tracker - obtain chain tracker
1939  * @ioc: per adapter object
1940  * @scmd: SCSI commands of the IO request
1941  *
1942  * Return: chain tracker from chain_lookup table using key as
1943  * smid and smid's chain_offset.
1944  */
1945 static struct chain_tracker *
1946 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
1947                                struct scsi_cmnd *scmd)
1948 {
1949         struct chain_tracker *chain_req;
1950         struct scsiio_tracker *st = scsi_cmd_priv(scmd);
1951         u16 smid = st->smid;
1952         u8 chain_offset =
1953            atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);
1954
1955         if (chain_offset == ioc->chains_needed_per_io)
1956                 return NULL;
1957
1958         chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];
1959         atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);
1960         return chain_req;
1961 }
1962
1963
1964 /**
1965  * _base_build_sg - build generic sg
1966  * @ioc: per adapter object
1967  * @psge: virtual address for SGE
1968  * @data_out_dma: physical address for WRITES
1969  * @data_out_sz: data xfer size for WRITES
1970  * @data_in_dma: physical address for READS
1971  * @data_in_sz: data xfer size for READS
1972  */
1973 static void
1974 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
1975         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1976         size_t data_in_sz)
1977 {
1978         u32 sgl_flags;
1979
1980         if (!data_out_sz && !data_in_sz) {
1981                 _base_build_zero_len_sge(ioc, psge);
1982                 return;
1983         }
1984
1985         if (data_out_sz && data_in_sz) {
1986                 /* WRITE sgel first */
1987                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1988                     MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
1989                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1990                 ioc->base_add_sg_single(psge, sgl_flags |
1991                     data_out_sz, data_out_dma);
1992
1993                 /* incr sgel */
1994                 psge += ioc->sge_size;
1995
1996                 /* READ sgel last */
1997                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1998                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1999                     MPI2_SGE_FLAGS_END_OF_LIST);
2000                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2001                 ioc->base_add_sg_single(psge, sgl_flags |
2002                     data_in_sz, data_in_dma);
2003         } else if (data_out_sz) /* WRITE */ {
2004                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2005                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
2006                     MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
2007                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2008                 ioc->base_add_sg_single(psge, sgl_flags |
2009                     data_out_sz, data_out_dma);
2010         } else if (data_in_sz) /* READ */ {
2011                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2012                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
2013                     MPI2_SGE_FLAGS_END_OF_LIST);
2014                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2015                 ioc->base_add_sg_single(psge, sgl_flags |
2016                     data_in_sz, data_in_dma);
2017         }
2018 }
2019
2020 /* IEEE format sgls */
2021
2022 /**
2023  * _base_build_nvme_prp - This function is called for NVMe end devices to build
2024  * a native SGL (NVMe PRP). The native SGL is built starting in the first PRP
2025  * entry of the NVMe message (PRP1).  If the data buffer is small enough to be
2026  * described entirely using PRP1, then PRP2 is not used.  If needed, PRP2 is
2027  * used to describe a larger data buffer.  If the data buffer is too large to
2028  * describe using the two PRP entriess inside the NVMe message, then PRP1
2029  * describes the first data memory segment, and PRP2 contains a pointer to a PRP
2030  * list located elsewhere in memory to describe the remaining data memory
2031  * segments.  The PRP list will be contiguous.
2032  *
2033  * The native SGL for NVMe devices is a Physical Region Page (PRP).  A PRP
2034  * consists of a list of PRP entries to describe a number of noncontigous
2035  * physical memory segments as a single memory buffer, just as a SGL does.  Note
2036  * however, that this function is only used by the IOCTL call, so the memory
2037  * given will be guaranteed to be contiguous.  There is no need to translate
2038  * non-contiguous SGL into a PRP in this case.  All PRPs will describe
2039  * contiguous space that is one page size each.
2040  *
2041  * Each NVMe message contains two PRP entries.  The first (PRP1) either contains
2042  * a PRP list pointer or a PRP element, depending upon the command.  PRP2
2043  * contains the second PRP element if the memory being described fits within 2
2044  * PRP entries, or a PRP list pointer if the PRP spans more than two entries.
2045  *
2046  * A PRP list pointer contains the address of a PRP list, structured as a linear
2047  * array of PRP entries.  Each PRP entry in this list describes a segment of
2048  * physical memory.
2049  *
2050  * Each 64-bit PRP entry comprises an address and an offset field.  The address
2051  * always points at the beginning of a 4KB physical memory page, and the offset
2052  * describes where within that 4KB page the memory segment begins.  Only the
2053  * first element in a PRP list may contain a non-zero offest, implying that all
2054  * memory segments following the first begin at the start of a 4KB page.
2055  *
2056  * Each PRP element normally describes 4KB of physical memory, with exceptions
2057  * for the first and last elements in the list.  If the memory being described
2058  * by the list begins at a non-zero offset within the first 4KB page, then the
2059  * first PRP element will contain a non-zero offset indicating where the region
2060  * begins within the 4KB page.  The last memory segment may end before the end
2061  * of the 4KB segment, depending upon the overall size of the memory being
2062  * described by the PRP list.
2063  *
2064  * Since PRP entries lack any indication of size, the overall data buffer length
2065  * is used to determine where the end of the data memory buffer is located, and
2066  * how many PRP entries are required to describe it.
2067  *
2068  * @ioc: per adapter object
2069  * @smid: system request message index for getting asscociated SGL
2070  * @nvme_encap_request: the NVMe request msg frame pointer
2071  * @data_out_dma: physical address for WRITES
2072  * @data_out_sz: data xfer size for WRITES
2073  * @data_in_dma: physical address for READS
2074  * @data_in_sz: data xfer size for READS
2075  */
2076 static void
2077 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,
2078         Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request,
2079         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2080         size_t data_in_sz)
2081 {
2082         int             prp_size = NVME_PRP_SIZE;
2083         __le64          *prp_entry, *prp1_entry, *prp2_entry;
2084         __le64          *prp_page;
2085         dma_addr_t      prp_entry_dma, prp_page_dma, dma_addr;
2086         u32             offset, entry_len;
2087         u32             page_mask_result, page_mask;
2088         size_t          length;
2089         struct mpt3sas_nvme_cmd *nvme_cmd =
2090                 (void *)nvme_encap_request->NVMe_Command;
2091
2092         /*
2093          * Not all commands require a data transfer. If no data, just return
2094          * without constructing any PRP.
2095          */
2096         if (!data_in_sz && !data_out_sz)
2097                 return;
2098         prp1_entry = &nvme_cmd->prp1;
2099         prp2_entry = &nvme_cmd->prp2;
2100         prp_entry = prp1_entry;
2101         /*
2102          * For the PRP entries, use the specially allocated buffer of
2103          * contiguous memory.
2104          */
2105         prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);
2106         prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
2107
2108         /*
2109          * Check if we are within 1 entry of a page boundary we don't
2110          * want our first entry to be a PRP List entry.
2111          */
2112         page_mask = ioc->page_size - 1;
2113         page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask;
2114         if (!page_mask_result) {
2115                 /* Bump up to next page boundary. */
2116                 prp_page = (__le64 *)((u8 *)prp_page + prp_size);
2117                 prp_page_dma = prp_page_dma + prp_size;
2118         }
2119
2120         /*
2121          * Set PRP physical pointer, which initially points to the current PRP
2122          * DMA memory page.
2123          */
2124         prp_entry_dma = prp_page_dma;
2125
2126         /* Get physical address and length of the data buffer. */
2127         if (data_in_sz) {
2128                 dma_addr = data_in_dma;
2129                 length = data_in_sz;
2130         } else {
2131                 dma_addr = data_out_dma;
2132                 length = data_out_sz;
2133         }
2134
2135         /* Loop while the length is not zero. */
2136         while (length) {
2137                 /*
2138                  * Check if we need to put a list pointer here if we are at
2139                  * page boundary - prp_size (8 bytes).
2140                  */
2141                 page_mask_result = (prp_entry_dma + prp_size) & page_mask;
2142                 if (!page_mask_result) {
2143                         /*
2144                          * This is the last entry in a PRP List, so we need to
2145                          * put a PRP list pointer here.  What this does is:
2146                          *   - bump the current memory pointer to the next
2147                          *     address, which will be the next full page.
2148                          *   - set the PRP Entry to point to that page.  This
2149                          *     is now the PRP List pointer.
2150                          *   - bump the PRP Entry pointer the start of the
2151                          *     next page.  Since all of this PRP memory is
2152                          *     contiguous, no need to get a new page - it's
2153                          *     just the next address.
2154                          */
2155                         prp_entry_dma++;
2156                         *prp_entry = cpu_to_le64(prp_entry_dma);
2157                         prp_entry++;
2158                 }
2159
2160                 /* Need to handle if entry will be part of a page. */
2161                 offset = dma_addr & page_mask;
2162                 entry_len = ioc->page_size - offset;
2163
2164                 if (prp_entry == prp1_entry) {
2165                         /*
2166                          * Must fill in the first PRP pointer (PRP1) before
2167                          * moving on.
2168                          */
2169                         *prp1_entry = cpu_to_le64(dma_addr);
2170
2171                         /*
2172                          * Now point to the second PRP entry within the
2173                          * command (PRP2).
2174                          */
2175                         prp_entry = prp2_entry;
2176                 } else if (prp_entry == prp2_entry) {
2177                         /*
2178                          * Should the PRP2 entry be a PRP List pointer or just
2179                          * a regular PRP pointer?  If there is more than one
2180                          * more page of data, must use a PRP List pointer.
2181                          */
2182                         if (length > ioc->page_size) {
2183                                 /*
2184                                  * PRP2 will contain a PRP List pointer because
2185                                  * more PRP's are needed with this command. The
2186                                  * list will start at the beginning of the
2187                                  * contiguous buffer.
2188                                  */
2189                                 *prp2_entry = cpu_to_le64(prp_entry_dma);
2190
2191                                 /*
2192                                  * The next PRP Entry will be the start of the
2193                                  * first PRP List.
2194                                  */
2195                                 prp_entry = prp_page;
2196                         } else {
2197                                 /*
2198                                  * After this, the PRP Entries are complete.
2199                                  * This command uses 2 PRP's and no PRP list.
2200                                  */
2201                                 *prp2_entry = cpu_to_le64(dma_addr);
2202                         }
2203                 } else {
2204                         /*
2205                          * Put entry in list and bump the addresses.
2206                          *
2207                          * After PRP1 and PRP2 are filled in, this will fill in
2208                          * all remaining PRP entries in a PRP List, one per
2209                          * each time through the loop.
2210                          */
2211                         *prp_entry = cpu_to_le64(dma_addr);
2212                         prp_entry++;
2213                         prp_entry_dma++;
2214                 }
2215
2216                 /*
2217                  * Bump the phys address of the command's data buffer by the
2218                  * entry_len.
2219                  */
2220                 dma_addr += entry_len;
2221
2222                 /* Decrement length accounting for last partial page. */
2223                 if (entry_len > length)
2224                         length = 0;
2225                 else
2226                         length -= entry_len;
2227         }
2228 }
2229
2230 /**
2231  * base_make_prp_nvme -
2232  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
2233  *
2234  * @ioc:                per adapter object
2235  * @scmd:               SCSI command from the mid-layer
2236  * @mpi_request:        mpi request
2237  * @smid:               msg Index
2238  * @sge_count:          scatter gather element count.
2239  *
2240  * Return:              true: PRPs are built
2241  *                      false: IEEE SGLs needs to be built
2242  */
2243 static void
2244 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,
2245                 struct scsi_cmnd *scmd,
2246                 Mpi25SCSIIORequest_t *mpi_request,
2247                 u16 smid, int sge_count)
2248 {
2249         int sge_len, num_prp_in_chain = 0;
2250         Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl;
2251         __le64 *curr_buff;
2252         dma_addr_t msg_dma, sge_addr, offset;
2253         u32 page_mask, page_mask_result;
2254         struct scatterlist *sg_scmd;
2255         u32 first_prp_len;
2256         int data_len = scsi_bufflen(scmd);
2257         u32 nvme_pg_size;
2258
2259         nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);
2260         /*
2261          * Nvme has a very convoluted prp format.  One prp is required
2262          * for each page or partial page. Driver need to split up OS sg_list
2263          * entries if it is longer than one page or cross a page
2264          * boundary.  Driver also have to insert a PRP list pointer entry as
2265          * the last entry in each physical page of the PRP list.
2266          *
2267          * NOTE: The first PRP "entry" is actually placed in the first
2268          * SGL entry in the main message as IEEE 64 format.  The 2nd
2269          * entry in the main message is the chain element, and the rest
2270          * of the PRP entries are built in the contiguous pcie buffer.
2271          */
2272         page_mask = nvme_pg_size - 1;
2273
2274         /*
2275          * Native SGL is needed.
2276          * Put a chain element in main message frame that points to the first
2277          * chain buffer.
2278          *
2279          * NOTE:  The ChainOffset field must be 0 when using a chain pointer to
2280          *        a native SGL.
2281          */
2282
2283         /* Set main message chain element pointer */
2284         main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2285         /*
2286          * For NVMe the chain element needs to be the 2nd SG entry in the main
2287          * message.
2288          */
2289         main_chain_element = (Mpi25IeeeSgeChain64_t *)
2290                 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
2291
2292         /*
2293          * For the PRP entries, use the specially allocated buffer of
2294          * contiguous memory.  Normal chain buffers can't be used
2295          * because each chain buffer would need to be the size of an OS
2296          * page (4k).
2297          */
2298         curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);
2299         msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
2300
2301         main_chain_element->Address = cpu_to_le64(msg_dma);
2302         main_chain_element->NextChainOffset = 0;
2303         main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2304                         MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2305                         MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2306
2307         /* Build first prp, sge need not to be page aligned*/
2308         ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2309         sg_scmd = scsi_sglist(scmd);
2310         sge_addr = sg_dma_address(sg_scmd);
2311         sge_len = sg_dma_len(sg_scmd);
2312
2313         offset = sge_addr & page_mask;
2314         first_prp_len = nvme_pg_size - offset;
2315
2316         ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2317         ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2318
2319         data_len -= first_prp_len;
2320
2321         if (sge_len > first_prp_len) {
2322                 sge_addr += first_prp_len;
2323                 sge_len -= first_prp_len;
2324         } else if (data_len && (sge_len == first_prp_len)) {
2325                 sg_scmd = sg_next(sg_scmd);
2326                 sge_addr = sg_dma_address(sg_scmd);
2327                 sge_len = sg_dma_len(sg_scmd);
2328         }
2329
2330         for (;;) {
2331                 offset = sge_addr & page_mask;
2332
2333                 /* Put PRP pointer due to page boundary*/
2334                 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask;
2335                 if (unlikely(!page_mask_result)) {
2336                         scmd_printk(KERN_NOTICE,
2337                                 scmd, "page boundary curr_buff: 0x%p\n",
2338                                 curr_buff);
2339                         msg_dma += 8;
2340                         *curr_buff = cpu_to_le64(msg_dma);
2341                         curr_buff++;
2342                         num_prp_in_chain++;
2343                 }
2344
2345                 *curr_buff = cpu_to_le64(sge_addr);
2346                 curr_buff++;
2347                 msg_dma += 8;
2348                 num_prp_in_chain++;
2349
2350                 sge_addr += nvme_pg_size;
2351                 sge_len -= nvme_pg_size;
2352                 data_len -= nvme_pg_size;
2353
2354                 if (data_len <= 0)
2355                         break;
2356
2357                 if (sge_len > 0)
2358                         continue;
2359
2360                 sg_scmd = sg_next(sg_scmd);
2361                 sge_addr = sg_dma_address(sg_scmd);
2362                 sge_len = sg_dma_len(sg_scmd);
2363         }
2364
2365         main_chain_element->Length =
2366                 cpu_to_le32(num_prp_in_chain * sizeof(u64));
2367         return;
2368 }
2369
2370 static bool
2371 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,
2372         struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count)
2373 {
2374         u32 data_length = 0;
2375         bool build_prp = true;
2376
2377         data_length = scsi_bufflen(scmd);
2378         if (pcie_device &&
2379             (mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))) {
2380                 build_prp = false;
2381                 return build_prp;
2382         }
2383
2384         /* If Datalenth is <= 16K and number of SGE’s entries are <= 2
2385          * we built IEEE SGL
2386          */
2387         if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2))
2388                 build_prp = false;
2389
2390         return build_prp;
2391 }
2392
2393 /**
2394  * _base_check_pcie_native_sgl - This function is called for PCIe end devices to
2395  * determine if the driver needs to build a native SGL.  If so, that native
2396  * SGL is built in the special contiguous buffers allocated especially for
2397  * PCIe SGL creation.  If the driver will not build a native SGL, return
2398  * TRUE and a normal IEEE SGL will be built.  Currently this routine
2399  * supports NVMe.
2400  * @ioc: per adapter object
2401  * @mpi_request: mf request pointer
2402  * @smid: system request message index
2403  * @scmd: scsi command
2404  * @pcie_device: points to the PCIe device's info
2405  *
2406  * Return: 0 if native SGL was built, 1 if no SGL was built
2407  */
2408 static int
2409 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,
2410         Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd,
2411         struct _pcie_device *pcie_device)
2412 {
2413         int sges_left;
2414
2415         /* Get the SG list pointer and info. */
2416         sges_left = scsi_dma_map(scmd);
2417         if (sges_left < 0) {
2418                 sdev_printk(KERN_ERR, scmd->device,
2419                         "scsi_dma_map failed: request for %d bytes!\n",
2420                         scsi_bufflen(scmd));
2421                 return 1;
2422         }
2423
2424         /* Check if we need to build a native SG list. */
2425         if (base_is_prp_possible(ioc, pcie_device,
2426                                 scmd, sges_left) == 0) {
2427                 /* We built a native SG list, just return. */
2428                 goto out;
2429         }
2430
2431         /*
2432          * Build native NVMe PRP.
2433          */
2434         base_make_prp_nvme(ioc, scmd, mpi_request,
2435                         smid, sges_left);
2436
2437         return 0;
2438 out:
2439         scsi_dma_unmap(scmd);
2440         return 1;
2441 }
2442
2443 /**
2444  * _base_add_sg_single_ieee - add sg element for IEEE format
2445  * @paddr: virtual address for SGE
2446  * @flags: SGE flags
2447  * @chain_offset: number of 128 byte elements from start of segment
2448  * @length: data transfer length
2449  * @dma_addr: Physical address
2450  */
2451 static void
2452 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
2453         dma_addr_t dma_addr)
2454 {
2455         Mpi25IeeeSgeChain64_t *sgel = paddr;
2456
2457         sgel->Flags = flags;
2458         sgel->NextChainOffset = chain_offset;
2459         sgel->Length = cpu_to_le32(length);
2460         sgel->Address = cpu_to_le64(dma_addr);
2461 }
2462
2463 /**
2464  * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
2465  * @ioc: per adapter object
2466  * @paddr: virtual address for SGE
2467  *
2468  * Create a zero length scatter gather entry to insure the IOCs hardware has
2469  * something to use if the target device goes brain dead and tries
2470  * to send data even when none is asked for.
2471  */
2472 static void
2473 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
2474 {
2475         u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2476                 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2477                 MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
2478
2479         _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
2480 }
2481
2482 /**
2483  * _base_build_sg_scmd - main sg creation routine
2484  *              pcie_device is unused here!
2485  * @ioc: per adapter object
2486  * @scmd: scsi command
2487  * @smid: system request message index
2488  * @unused: unused pcie_device pointer
2489  * Context: none.
2490  *
2491  * The main routine that builds scatter gather table from a given
2492  * scsi request sent via the .queuecommand main handler.
2493  *
2494  * Return: 0 success, anything else error
2495  */
2496 static int
2497 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
2498         struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused)
2499 {
2500         Mpi2SCSIIORequest_t *mpi_request;
2501         dma_addr_t chain_dma;
2502         struct scatterlist *sg_scmd;
2503         void *sg_local, *chain;
2504         u32 chain_offset;
2505         u32 chain_length;
2506         u32 chain_flags;
2507         int sges_left;
2508         u32 sges_in_segment;
2509         u32 sgl_flags;
2510         u32 sgl_flags_last_element;
2511         u32 sgl_flags_end_buffer;
2512         struct chain_tracker *chain_req;
2513
2514         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2515
2516         /* init scatter gather flags */
2517         sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
2518         if (scmd->sc_data_direction == DMA_TO_DEVICE)
2519                 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2520         sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
2521             << MPI2_SGE_FLAGS_SHIFT;
2522         sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
2523             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
2524             << MPI2_SGE_FLAGS_SHIFT;
2525         sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2526
2527         sg_scmd = scsi_sglist(scmd);
2528         sges_left = scsi_dma_map(scmd);
2529         if (sges_left < 0) {
2530                 sdev_printk(KERN_ERR, scmd->device,
2531                  "scsi_dma_map failed: request for %d bytes!\n",
2532                  scsi_bufflen(scmd));
2533                 return -ENOMEM;
2534         }
2535
2536         sg_local = &mpi_request->SGL;
2537         sges_in_segment = ioc->max_sges_in_main_message;
2538         if (sges_left <= sges_in_segment)
2539                 goto fill_in_last_segment;
2540
2541         mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
2542             (sges_in_segment * ioc->sge_size))/4;
2543
2544         /* fill in main message segment when there is a chain following */
2545         while (sges_in_segment) {
2546                 if (sges_in_segment == 1)
2547                         ioc->base_add_sg_single(sg_local,
2548                             sgl_flags_last_element | sg_dma_len(sg_scmd),
2549                             sg_dma_address(sg_scmd));
2550                 else
2551                         ioc->base_add_sg_single(sg_local, sgl_flags |
2552                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2553                 sg_scmd = sg_next(sg_scmd);
2554                 sg_local += ioc->sge_size;
2555                 sges_left--;
2556                 sges_in_segment--;
2557         }
2558
2559         /* initializing the chain flags and pointers */
2560         chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
2561         chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2562         if (!chain_req)
2563                 return -1;
2564         chain = chain_req->chain_buffer;
2565         chain_dma = chain_req->chain_buffer_dma;
2566         do {
2567                 sges_in_segment = (sges_left <=
2568                     ioc->max_sges_in_chain_message) ? sges_left :
2569                     ioc->max_sges_in_chain_message;
2570                 chain_offset = (sges_left == sges_in_segment) ?
2571                     0 : (sges_in_segment * ioc->sge_size)/4;
2572                 chain_length = sges_in_segment * ioc->sge_size;
2573                 if (chain_offset) {
2574                         chain_offset = chain_offset <<
2575                             MPI2_SGE_CHAIN_OFFSET_SHIFT;
2576                         chain_length += ioc->sge_size;
2577                 }
2578                 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
2579                     chain_length, chain_dma);
2580                 sg_local = chain;
2581                 if (!chain_offset)
2582                         goto fill_in_last_segment;
2583
2584                 /* fill in chain segments */
2585                 while (sges_in_segment) {
2586                         if (sges_in_segment == 1)
2587                                 ioc->base_add_sg_single(sg_local,
2588                                     sgl_flags_last_element |
2589                                     sg_dma_len(sg_scmd),
2590                                     sg_dma_address(sg_scmd));
2591                         else
2592                                 ioc->base_add_sg_single(sg_local, sgl_flags |
2593                                     sg_dma_len(sg_scmd),
2594                                     sg_dma_address(sg_scmd));
2595                         sg_scmd = sg_next(sg_scmd);
2596                         sg_local += ioc->sge_size;
2597                         sges_left--;
2598                         sges_in_segment--;
2599                 }
2600
2601                 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2602                 if (!chain_req)
2603                         return -1;
2604                 chain = chain_req->chain_buffer;
2605                 chain_dma = chain_req->chain_buffer_dma;
2606         } while (1);
2607
2608
2609  fill_in_last_segment:
2610
2611         /* fill the last segment */
2612         while (sges_left) {
2613                 if (sges_left == 1)
2614                         ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
2615                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2616                 else
2617                         ioc->base_add_sg_single(sg_local, sgl_flags |
2618                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2619                 sg_scmd = sg_next(sg_scmd);
2620                 sg_local += ioc->sge_size;
2621                 sges_left--;
2622         }
2623
2624         return 0;
2625 }
2626
2627 /**
2628  * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
2629  * @ioc: per adapter object
2630  * @scmd: scsi command
2631  * @smid: system request message index
2632  * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be
2633  * constructed on need.
2634  * Context: none.
2635  *
2636  * The main routine that builds scatter gather table from a given
2637  * scsi request sent via the .queuecommand main handler.
2638  *
2639  * Return: 0 success, anything else error
2640  */
2641 static int
2642 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
2643         struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device)
2644 {
2645         Mpi25SCSIIORequest_t *mpi_request;
2646         dma_addr_t chain_dma;
2647         struct scatterlist *sg_scmd;
2648         void *sg_local, *chain;
2649         u32 chain_offset;
2650         u32 chain_length;
2651         int sges_left;
2652         u32 sges_in_segment;
2653         u8 simple_sgl_flags;
2654         u8 simple_sgl_flags_last;
2655         u8 chain_sgl_flags;
2656         struct chain_tracker *chain_req;
2657
2658         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2659
2660         /* init scatter gather flags */
2661         simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2662             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2663         simple_sgl_flags_last = simple_sgl_flags |
2664             MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2665         chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2666             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2667
2668         /* Check if we need to build a native SG list. */
2669         if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,
2670                         smid, scmd, pcie_device) == 0)) {
2671                 /* We built a native SG list, just return. */
2672                 return 0;
2673         }
2674
2675         sg_scmd = scsi_sglist(scmd);
2676         sges_left = scsi_dma_map(scmd);
2677         if (sges_left < 0) {
2678                 sdev_printk(KERN_ERR, scmd->device,
2679                         "scsi_dma_map failed: request for %d bytes!\n",
2680                         scsi_bufflen(scmd));
2681                 return -ENOMEM;
2682         }
2683
2684         sg_local = &mpi_request->SGL;
2685         sges_in_segment = (ioc->request_sz -
2686                    offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
2687         if (sges_left <= sges_in_segment)
2688                 goto fill_in_last_segment;
2689
2690         mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
2691             (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
2692
2693         /* fill in main message segment when there is a chain following */
2694         while (sges_in_segment > 1) {
2695                 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2696                     sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2697                 sg_scmd = sg_next(sg_scmd);
2698                 sg_local += ioc->sge_size_ieee;
2699                 sges_left--;
2700                 sges_in_segment--;
2701         }
2702
2703         /* initializing the pointers */
2704         chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2705         if (!chain_req)
2706                 return -1;
2707         chain = chain_req->chain_buffer;
2708         chain_dma = chain_req->chain_buffer_dma;
2709         do {
2710                 sges_in_segment = (sges_left <=
2711                     ioc->max_sges_in_chain_message) ? sges_left :
2712                     ioc->max_sges_in_chain_message;
2713                 chain_offset = (sges_left == sges_in_segment) ?
2714                     0 : sges_in_segment;
2715                 chain_length = sges_in_segment * ioc->sge_size_ieee;
2716                 if (chain_offset)
2717                         chain_length += ioc->sge_size_ieee;
2718                 _base_add_sg_single_ieee(sg_local, chain_sgl_flags,
2719                     chain_offset, chain_length, chain_dma);
2720
2721                 sg_local = chain;
2722                 if (!chain_offset)
2723                         goto fill_in_last_segment;
2724
2725                 /* fill in chain segments */
2726                 while (sges_in_segment) {
2727                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2728                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2729                         sg_scmd = sg_next(sg_scmd);
2730                         sg_local += ioc->sge_size_ieee;
2731                         sges_left--;
2732                         sges_in_segment--;
2733                 }
2734
2735                 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2736                 if (!chain_req)
2737                         return -1;
2738                 chain = chain_req->chain_buffer;
2739                 chain_dma = chain_req->chain_buffer_dma;
2740         } while (1);
2741
2742
2743  fill_in_last_segment:
2744
2745         /* fill the last segment */
2746         while (sges_left > 0) {
2747                 if (sges_left == 1)
2748                         _base_add_sg_single_ieee(sg_local,
2749                             simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
2750                             sg_dma_address(sg_scmd));
2751                 else
2752                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2753                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2754                 sg_scmd = sg_next(sg_scmd);
2755                 sg_local += ioc->sge_size_ieee;
2756                 sges_left--;
2757         }
2758
2759         return 0;
2760 }
2761
2762 /**
2763  * _base_build_sg_ieee - build generic sg for IEEE format
2764  * @ioc: per adapter object
2765  * @psge: virtual address for SGE
2766  * @data_out_dma: physical address for WRITES
2767  * @data_out_sz: data xfer size for WRITES
2768  * @data_in_dma: physical address for READS
2769  * @data_in_sz: data xfer size for READS
2770  */
2771 static void
2772 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
2773         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2774         size_t data_in_sz)
2775 {
2776         u8 sgl_flags;
2777
2778         if (!data_out_sz && !data_in_sz) {
2779                 _base_build_zero_len_sge_ieee(ioc, psge);
2780                 return;
2781         }
2782
2783         if (data_out_sz && data_in_sz) {
2784                 /* WRITE sgel first */
2785                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2786                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2787                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2788                     data_out_dma);
2789
2790                 /* incr sgel */
2791                 psge += ioc->sge_size_ieee;
2792
2793                 /* READ sgel last */
2794                 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2795                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2796                     data_in_dma);
2797         } else if (data_out_sz) /* WRITE */ {
2798                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2799                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2800                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2801                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2802                     data_out_dma);
2803         } else if (data_in_sz) /* READ */ {
2804                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2805                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2806                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2807                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2808                     data_in_dma);
2809         }
2810 }
2811
2812 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
2813
2814 /**
2815  * _base_config_dma_addressing - set dma addressing
2816  * @ioc: per adapter object
2817  * @pdev: PCI device struct
2818  *
2819  * Return: 0 for success, non-zero for failure.
2820  */
2821 static int
2822 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
2823 {
2824         struct sysinfo s;
2825
2826         if (ioc->is_mcpu_endpoint ||
2827             sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma ||
2828             dma_get_required_mask(&pdev->dev) <= DMA_BIT_MASK(32))
2829                 ioc->dma_mask = 32;
2830         /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */
2831         else if (ioc->hba_mpi_version_belonged > MPI2_VERSION)
2832                 ioc->dma_mask = 63;
2833         else
2834                 ioc->dma_mask = 64;
2835
2836         if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)) ||
2837             dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)))
2838                 return -ENODEV;
2839
2840         if (ioc->dma_mask > 32) {
2841                 ioc->base_add_sg_single = &_base_add_sg_single_64;
2842                 ioc->sge_size = sizeof(Mpi2SGESimple64_t);
2843         } else {
2844                 ioc->base_add_sg_single = &_base_add_sg_single_32;
2845                 ioc->sge_size = sizeof(Mpi2SGESimple32_t);
2846         }
2847
2848         si_meminfo(&s);
2849         ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
2850                 ioc->dma_mask, convert_to_kb(s.totalram));
2851
2852         return 0;
2853 }
2854
2855 /**
2856  * _base_check_enable_msix - checks MSIX capabable.
2857  * @ioc: per adapter object
2858  *
2859  * Check to see if card is capable of MSIX, and set number
2860  * of available msix vectors
2861  */
2862 static int
2863 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2864 {
2865         int base;
2866         u16 message_control;
2867
2868         /* Check whether controller SAS2008 B0 controller,
2869          * if it is SAS2008 B0 controller use IO-APIC instead of MSIX
2870          */
2871         if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&
2872             ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {
2873                 return -EINVAL;
2874         }
2875
2876         base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
2877         if (!base) {
2878                 dfailprintk(ioc, ioc_info(ioc, "msix not supported\n"));
2879                 return -EINVAL;
2880         }
2881
2882         /* get msix vector count */
2883         /* NUMA_IO not supported for older controllers */
2884         if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||
2885             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||
2886             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||
2887             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||
2888             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||
2889             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||
2890             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)
2891                 ioc->msix_vector_count = 1;
2892         else {
2893                 pci_read_config_word(ioc->pdev, base + 2, &message_control);
2894                 ioc->msix_vector_count = (message_control & 0x3FF) + 1;
2895         }
2896         dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n",
2897                                   ioc->msix_vector_count));
2898         return 0;
2899 }
2900
2901 /**
2902  * _base_free_irq - free irq
2903  * @ioc: per adapter object
2904  *
2905  * Freeing respective reply_queue from the list.
2906  */
2907 static void
2908 _base_free_irq(struct MPT3SAS_ADAPTER *ioc)
2909 {
2910         struct adapter_reply_queue *reply_q, *next;
2911
2912         if (list_empty(&ioc->reply_queue_list))
2913                 return;
2914
2915         list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
2916                 list_del(&reply_q->list);
2917                 if (ioc->smp_affinity_enable)
2918                         irq_set_affinity_hint(pci_irq_vector(ioc->pdev,
2919                             reply_q->msix_index), NULL);
2920                 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
2921                          reply_q);
2922                 kfree(reply_q);
2923         }
2924 }
2925
2926 /**
2927  * _base_request_irq - request irq
2928  * @ioc: per adapter object
2929  * @index: msix index into vector table
2930  *
2931  * Inserting respective reply_queue into the list.
2932  */
2933 static int
2934 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
2935 {
2936         struct pci_dev *pdev = ioc->pdev;
2937         struct adapter_reply_queue *reply_q;
2938         int r;
2939
2940         reply_q =  kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
2941         if (!reply_q) {
2942                 ioc_err(ioc, "unable to allocate memory %zu!\n",
2943                         sizeof(struct adapter_reply_queue));
2944                 return -ENOMEM;
2945         }
2946         reply_q->ioc = ioc;
2947         reply_q->msix_index = index;
2948
2949         atomic_set(&reply_q->busy, 0);
2950         if (ioc->msix_enable)
2951                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
2952                     ioc->driver_name, ioc->id, index);
2953         else
2954                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
2955                     ioc->driver_name, ioc->id);
2956         r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
2957                         IRQF_SHARED, reply_q->name, reply_q);
2958         if (r) {
2959                 pr_err("%s: unable to allocate interrupt %d!\n",
2960                        reply_q->name, pci_irq_vector(pdev, index));
2961                 kfree(reply_q);
2962                 return -EBUSY;
2963         }
2964
2965         INIT_LIST_HEAD(&reply_q->list);
2966         list_add_tail(&reply_q->list, &ioc->reply_queue_list);
2967         return 0;
2968 }
2969
2970 /**
2971  * _base_assign_reply_queues - assigning msix index for each cpu
2972  * @ioc: per adapter object
2973  *
2974  * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
2975  *
2976  * It would nice if we could call irq_set_affinity, however it is not
2977  * an exported symbol
2978  */
2979 static void
2980 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
2981 {
2982         unsigned int cpu, nr_cpus, nr_msix, index = 0;
2983         struct adapter_reply_queue *reply_q;
2984         int local_numa_node;
2985
2986         if (!_base_is_controller_msix_enabled(ioc))
2987                 return;
2988
2989         if (ioc->msix_load_balance)
2990                 return;
2991
2992         memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
2993
2994         nr_cpus = num_online_cpus();
2995         nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
2996                                                ioc->facts.MaxMSIxVectors);
2997         if (!nr_msix)
2998                 return;
2999
3000         if (ioc->smp_affinity_enable) {
3001
3002                 /*
3003                  * set irq affinity to local numa node for those irqs
3004                  * corresponding to high iops queues.
3005                  */
3006                 if (ioc->high_iops_queues) {
3007                         local_numa_node = dev_to_node(&ioc->pdev->dev);
3008                         for (index = 0; index < ioc->high_iops_queues;
3009                             index++) {
3010                                 irq_set_affinity_hint(pci_irq_vector(ioc->pdev,
3011                                     index), cpumask_of_node(local_numa_node));
3012                         }
3013                 }
3014
3015                 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
3016                         const cpumask_t *mask;
3017
3018                         if (reply_q->msix_index < ioc->high_iops_queues)
3019                                 continue;
3020
3021                         mask = pci_irq_get_affinity(ioc->pdev,
3022                             reply_q->msix_index);
3023                         if (!mask) {
3024                                 ioc_warn(ioc, "no affinity for msi %x\n",
3025                                          reply_q->msix_index);
3026                                 goto fall_back;
3027                         }
3028
3029                         for_each_cpu_and(cpu, mask, cpu_online_mask) {
3030                                 if (cpu >= ioc->cpu_msix_table_sz)
3031                                         break;
3032                                 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
3033                         }
3034                 }
3035                 return;
3036         }
3037
3038 fall_back:
3039         cpu = cpumask_first(cpu_online_mask);
3040         nr_msix -= ioc->high_iops_queues;
3041         index = 0;
3042
3043         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
3044                 unsigned int i, group = nr_cpus / nr_msix;
3045
3046                 if (reply_q->msix_index < ioc->high_iops_queues)
3047                         continue;
3048
3049                 if (cpu >= nr_cpus)
3050                         break;
3051
3052                 if (index < nr_cpus % nr_msix)
3053                         group++;
3054
3055                 for (i = 0 ; i < group ; i++) {
3056                         ioc->cpu_msix_table[cpu] = reply_q->msix_index;
3057                         cpu = cpumask_next(cpu, cpu_online_mask);
3058                 }
3059                 index++;
3060         }
3061 }
3062
3063 /**
3064  * _base_check_and_enable_high_iops_queues - enable high iops mode
3065  * @ioc: per adapter object
3066  * @hba_msix_vector_count: msix vectors supported by HBA
3067  *
3068  * Enable high iops queues only if
3069  *  - HBA is a SEA/AERO controller and
3070  *  - MSI-Xs vector supported by the HBA is 128 and
3071  *  - total CPU count in the system >=16 and
3072  *  - loaded driver with default max_msix_vectors module parameter and
3073  *  - system booted in non kdump mode
3074  *
3075  * returns nothing.
3076  */
3077 static void
3078 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc,
3079                 int hba_msix_vector_count)
3080 {
3081         u16 lnksta, speed;
3082
3083         if (perf_mode == MPT_PERF_MODE_IOPS ||
3084             perf_mode == MPT_PERF_MODE_LATENCY) {
3085                 ioc->high_iops_queues = 0;
3086                 return;
3087         }
3088
3089         if (perf_mode == MPT_PERF_MODE_DEFAULT) {
3090
3091                 pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta);
3092                 speed = lnksta & PCI_EXP_LNKSTA_CLS;
3093
3094                 if (speed < 0x4) {
3095                         ioc->high_iops_queues = 0;
3096                         return;
3097                 }
3098         }
3099
3100         if (!reset_devices && ioc->is_aero_ioc &&
3101             hba_msix_vector_count == MPT3SAS_GEN35_MAX_MSIX_QUEUES &&
3102             num_online_cpus() >= MPT3SAS_HIGH_IOPS_REPLY_QUEUES &&
3103             max_msix_vectors == -1)
3104                 ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES;
3105         else
3106                 ioc->high_iops_queues = 0;
3107 }
3108
3109 /**
3110  * _base_disable_msix - disables msix
3111  * @ioc: per adapter object
3112  *
3113  */
3114 static void
3115 _base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
3116 {
3117         if (!ioc->msix_enable)
3118                 return;
3119         pci_free_irq_vectors(ioc->pdev);
3120         ioc->msix_enable = 0;
3121 }
3122
3123 /**
3124  * _base_alloc_irq_vectors - allocate msix vectors
3125  * @ioc: per adapter object
3126  *
3127  */
3128 static int
3129 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc)
3130 {
3131         int i, irq_flags = PCI_IRQ_MSIX;
3132         struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues };
3133         struct irq_affinity *descp = &desc;
3134
3135         if (ioc->smp_affinity_enable)
3136                 irq_flags |= PCI_IRQ_AFFINITY;
3137         else
3138                 descp = NULL;
3139
3140         ioc_info(ioc, " %d %d\n", ioc->high_iops_queues,
3141             ioc->reply_queue_count);
3142
3143         i = pci_alloc_irq_vectors_affinity(ioc->pdev,
3144             ioc->high_iops_queues,
3145             ioc->reply_queue_count, irq_flags, descp);
3146
3147         return i;
3148 }
3149
3150 /**
3151  * _base_enable_msix - enables msix, failback to io_apic
3152  * @ioc: per adapter object
3153  *
3154  */
3155 static int
3156 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
3157 {
3158         int r;
3159         int i, local_max_msix_vectors;
3160         u8 try_msix = 0;
3161
3162         ioc->msix_load_balance = false;
3163
3164         if (msix_disable == -1 || msix_disable == 0)
3165                 try_msix = 1;
3166
3167         if (!try_msix)
3168                 goto try_ioapic;
3169
3170         if (_base_check_enable_msix(ioc) != 0)
3171                 goto try_ioapic;
3172
3173         ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count);
3174         pr_info("\t no of cores: %d, max_msix_vectors: %d\n",
3175                 ioc->cpu_count, max_msix_vectors);
3176         if (ioc->is_aero_ioc)
3177                 _base_check_and_enable_high_iops_queues(ioc,
3178                         ioc->msix_vector_count);
3179         ioc->reply_queue_count =
3180                 min_t(int, ioc->cpu_count + ioc->high_iops_queues,
3181                 ioc->msix_vector_count);
3182
3183         if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
3184                 local_max_msix_vectors = (reset_devices) ? 1 : 8;
3185         else
3186                 local_max_msix_vectors = max_msix_vectors;
3187
3188         if (local_max_msix_vectors > 0)
3189                 ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
3190                         ioc->reply_queue_count);
3191         else if (local_max_msix_vectors == 0)
3192                 goto try_ioapic;
3193
3194         /*
3195          * Enable msix_load_balance only if combined reply queue mode is
3196          * disabled on SAS3 & above generation HBA devices.
3197          */
3198         if (!ioc->combined_reply_queue &&
3199             ioc->hba_mpi_version_belonged != MPI2_VERSION) {
3200                 ioc_info(ioc,
3201                     "combined ReplyQueue is off, Enabling msix load balance\n");
3202                 ioc->msix_load_balance = true;
3203         }
3204
3205         /*
3206          * smp affinity setting is not need when msix load balance
3207          * is enabled.
3208          */
3209         if (ioc->msix_load_balance)
3210                 ioc->smp_affinity_enable = 0;
3211
3212         r = _base_alloc_irq_vectors(ioc);
3213         if (r < 0) {
3214                 ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r);
3215                 goto try_ioapic;
3216         }
3217
3218         ioc->msix_enable = 1;
3219         ioc->reply_queue_count = r;
3220         for (i = 0; i < ioc->reply_queue_count; i++) {
3221                 r = _base_request_irq(ioc, i);
3222                 if (r) {
3223                         _base_free_irq(ioc);
3224                         _base_disable_msix(ioc);
3225                         goto try_ioapic;
3226                 }
3227         }
3228
3229         ioc_info(ioc, "High IOPs queues : %s\n",
3230                         ioc->high_iops_queues ? "enabled" : "disabled");
3231
3232         return 0;
3233
3234 /* failback to io_apic interrupt routing */
3235  try_ioapic:
3236         ioc->high_iops_queues = 0;
3237         ioc_info(ioc, "High IOPs queues : disabled\n");
3238         ioc->reply_queue_count = 1;
3239         r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
3240         if (r < 0) {
3241                 dfailprintk(ioc,
3242                             ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
3243                                      r));
3244         } else
3245                 r = _base_request_irq(ioc, 0);
3246
3247         return r;
3248 }
3249
3250 /**
3251  * mpt3sas_base_unmap_resources - free controller resources
3252  * @ioc: per adapter object
3253  */
3254 static void
3255 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
3256 {
3257         struct pci_dev *pdev = ioc->pdev;
3258
3259         dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
3260
3261         _base_free_irq(ioc);
3262         _base_disable_msix(ioc);
3263
3264         kfree(ioc->replyPostRegisterIndex);
3265         ioc->replyPostRegisterIndex = NULL;
3266
3267
3268         if (ioc->chip_phys) {
3269                 iounmap(ioc->chip);
3270                 ioc->chip_phys = 0;
3271         }
3272
3273         if (pci_is_enabled(pdev)) {
3274                 pci_release_selected_regions(ioc->pdev, ioc->bars);
3275                 pci_disable_pcie_error_reporting(pdev);
3276                 pci_disable_device(pdev);
3277         }
3278 }
3279
3280 static int
3281 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
3282
3283 /**
3284  * _base_check_for_fault_and_issue_reset - check if IOC is in fault state
3285  *     and if it is in fault state then issue diag reset.
3286  * @ioc: per adapter object
3287  *
3288  * Returns: 0 for success, non-zero for failure.
3289  */
3290 static int
3291 _base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc)
3292 {
3293         u32 ioc_state;
3294         int rc = -EFAULT;
3295
3296         dinitprintk(ioc, pr_info("%s\n", __func__));
3297         if (ioc->pci_error_recovery)
3298                 return 0;
3299         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
3300         dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state));
3301
3302         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
3303                 mpt3sas_print_fault_code(ioc, ioc_state &
3304                     MPI2_DOORBELL_DATA_MASK);
3305                 rc = _base_diag_reset(ioc);
3306         } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
3307             MPI2_IOC_STATE_COREDUMP) {
3308                 mpt3sas_print_coredump_info(ioc, ioc_state &
3309                      MPI2_DOORBELL_DATA_MASK);
3310                 mpt3sas_base_wait_for_coredump_completion(ioc, __func__);
3311                 rc = _base_diag_reset(ioc);
3312         }
3313
3314         return rc;
3315 }
3316
3317 /**
3318  * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
3319  * @ioc: per adapter object
3320  *
3321  * Return: 0 for success, non-zero for failure.
3322  */
3323 int
3324 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
3325 {
3326         struct pci_dev *pdev = ioc->pdev;
3327         u32 memap_sz;
3328         u32 pio_sz;
3329         int i, r = 0, rc;
3330         u64 pio_chip = 0;
3331         phys_addr_t chip_phys = 0;
3332         struct adapter_reply_queue *reply_q;
3333
3334         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
3335
3336         ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3337         if (pci_enable_device_mem(pdev)) {
3338                 ioc_warn(ioc, "pci_enable_device_mem: failed\n");
3339                 ioc->bars = 0;
3340                 return -ENODEV;
3341         }
3342
3343
3344         if (pci_request_selected_regions(pdev, ioc->bars,
3345             ioc->driver_name)) {
3346                 ioc_warn(ioc, "pci_request_selected_regions: failed\n");
3347                 ioc->bars = 0;
3348                 r = -ENODEV;
3349                 goto out_fail;
3350         }
3351
3352 /* AER (Advanced Error Reporting) hooks */
3353         pci_enable_pcie_error_reporting(pdev);
3354
3355         pci_set_master(pdev);
3356
3357
3358         if (_base_config_dma_addressing(ioc, pdev) != 0) {
3359                 ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev));
3360                 r = -ENODEV;
3361                 goto out_fail;
3362         }
3363
3364         for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
3365              (!memap_sz || !pio_sz); i++) {
3366                 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
3367                         if (pio_sz)
3368                                 continue;
3369                         pio_chip = (u64)pci_resource_start(pdev, i);
3370                         pio_sz = pci_resource_len(pdev, i);
3371                 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3372                         if (memap_sz)
3373                                 continue;
3374                         ioc->chip_phys = pci_resource_start(pdev, i);
3375                         chip_phys = ioc->chip_phys;
3376                         memap_sz = pci_resource_len(pdev, i);
3377                         ioc->chip = ioremap(ioc->chip_phys, memap_sz);
3378                 }
3379         }
3380
3381         if (ioc->chip == NULL) {
3382                 ioc_err(ioc,
3383                     "unable to map adapter memory! or resource not found\n");
3384                 r = -EINVAL;
3385                 goto out_fail;
3386         }
3387
3388         mpt3sas_base_mask_interrupts(ioc);
3389
3390         r = _base_get_ioc_facts(ioc);
3391         if (r) {
3392                 rc = _base_check_for_fault_and_issue_reset(ioc);
3393                 if (rc || (_base_get_ioc_facts(ioc)))
3394                         goto out_fail;
3395         }
3396
3397         if (!ioc->rdpq_array_enable_assigned) {
3398                 ioc->rdpq_array_enable = ioc->rdpq_array_capable;
3399                 ioc->rdpq_array_enable_assigned = 1;
3400         }
3401
3402         r = _base_enable_msix(ioc);
3403         if (r)
3404                 goto out_fail;
3405
3406         if (!ioc->is_driver_loading)
3407                 _base_init_irqpolls(ioc);
3408         /* Use the Combined reply queue feature only for SAS3 C0 & higher
3409          * revision HBAs and also only when reply queue count is greater than 8
3410          */
3411         if (ioc->combined_reply_queue) {
3412                 /* Determine the Supplemental Reply Post Host Index Registers
3413                  * Addresse. Supplemental Reply Post Host Index Registers
3414                  * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
3415                  * each register is at offset bytes of
3416                  * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
3417                  */
3418                 ioc->replyPostRegisterIndex = kcalloc(
3419                      ioc->combined_reply_index_count,
3420                      sizeof(resource_size_t *), GFP_KERNEL);
3421                 if (!ioc->replyPostRegisterIndex) {
3422                         ioc_err(ioc,
3423                             "allocation for replyPostRegisterIndex failed!\n");
3424                         r = -ENOMEM;
3425                         goto out_fail;
3426                 }
3427
3428                 for (i = 0; i < ioc->combined_reply_index_count; i++) {
3429                         ioc->replyPostRegisterIndex[i] = (resource_size_t *)
3430                              ((u8 __force *)&ioc->chip->Doorbell +
3431                              MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
3432                              (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
3433                 }
3434         }
3435
3436         if (ioc->is_warpdrive) {
3437                 ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
3438                     &ioc->chip->ReplyPostHostIndex;
3439
3440                 for (i = 1; i < ioc->cpu_msix_table_sz; i++)
3441                         ioc->reply_post_host_index[i] =
3442                         (resource_size_t __iomem *)
3443                         ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
3444                         * 4)));
3445         }
3446
3447         list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
3448                 pr_info("%s: %s enabled: IRQ %d\n",
3449                         reply_q->name,
3450                         ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC",
3451                         pci_irq_vector(ioc->pdev, reply_q->msix_index));
3452
3453         ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n",
3454                  &chip_phys, ioc->chip, memap_sz);
3455         ioc_info(ioc, "ioport(0x%016llx), size(%d)\n",
3456                  (unsigned long long)pio_chip, pio_sz);
3457
3458         /* Save PCI configuration state for recovery from PCI AER/EEH errors */
3459         pci_save_state(pdev);
3460         return 0;
3461
3462  out_fail:
3463         mpt3sas_base_unmap_resources(ioc);
3464         return r;
3465 }
3466
3467 /**
3468  * mpt3sas_base_get_msg_frame - obtain request mf pointer
3469  * @ioc: per adapter object
3470  * @smid: system request message index(smid zero is invalid)
3471  *
3472  * Return: virt pointer to message frame.
3473  */
3474 void *
3475 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3476 {
3477         return (void *)(ioc->request + (smid * ioc->request_sz));
3478 }
3479
3480 /**
3481  * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
3482  * @ioc: per adapter object
3483  * @smid: system request message index
3484  *
3485  * Return: virt pointer to sense buffer.
3486  */
3487 void *
3488 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3489 {
3490         return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
3491 }
3492
3493 /**
3494  * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
3495  * @ioc: per adapter object
3496  * @smid: system request message index
3497  *
3498  * Return: phys pointer to the low 32bit address of the sense buffer.
3499  */
3500 __le32
3501 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3502 {
3503         return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
3504             SCSI_SENSE_BUFFERSIZE));
3505 }
3506
3507 /**
3508  * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr
3509  * @ioc: per adapter object
3510  * @smid: system request message index
3511  *
3512  * Return: virt pointer to a PCIe SGL.
3513  */
3514 void *
3515 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3516 {
3517         return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);
3518 }
3519
3520 /**
3521  * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr
3522  * @ioc: per adapter object
3523  * @smid: system request message index
3524  *
3525  * Return: phys pointer to the address of the PCIe buffer.
3526  */
3527 dma_addr_t
3528 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3529 {
3530         return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;
3531 }
3532
3533 /**
3534  * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
3535  * @ioc: per adapter object
3536  * @phys_addr: lower 32 physical addr of the reply
3537  *
3538  * Converts 32bit lower physical addr into a virt address.
3539  */
3540 void *
3541 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
3542 {
3543         if (!phys_addr)
3544                 return NULL;
3545         return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
3546 }
3547
3548 /**
3549  * _base_get_msix_index - get the msix index
3550  * @ioc: per adapter object
3551  * @scmd: scsi_cmnd object
3552  *
3553  * returns msix index of general reply queues,
3554  * i.e. reply queue on which IO request's reply
3555  * should be posted by the HBA firmware.
3556  */
3557 static inline u8
3558 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc,
3559         struct scsi_cmnd *scmd)
3560 {
3561         /* Enables reply_queue load balancing */
3562         if (ioc->msix_load_balance)
3563                 return ioc->reply_queue_count ?
3564                     base_mod64(atomic64_add_return(1,
3565                     &ioc->total_io_cnt), ioc->reply_queue_count) : 0;
3566
3567         return ioc->cpu_msix_table[raw_smp_processor_id()];
3568 }
3569
3570 /**
3571  * _base_sdev_nr_inflight_request -get number of inflight requests
3572  *                                 of a request queue.
3573  * @q: request_queue object
3574  *
3575  * returns number of inflight request of a request queue.
3576  */
3577 inline unsigned long
3578 _base_sdev_nr_inflight_request(struct request_queue *q)
3579 {
3580         struct blk_mq_hw_ctx *hctx = q->queue_hw_ctx[0];
3581
3582         return atomic_read(&hctx->nr_active);
3583 }
3584
3585
3586 /**
3587  * _base_get_high_iops_msix_index - get the msix index of
3588  *                              high iops queues
3589  * @ioc: per adapter object
3590  * @scmd: scsi_cmnd object
3591  *
3592  * Returns: msix index of high iops reply queues.
3593  * i.e. high iops reply queue on which IO request's
3594  * reply should be posted by the HBA firmware.
3595  */
3596 static inline u8
3597 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc,
3598         struct scsi_cmnd *scmd)
3599 {
3600         /**
3601          * Round robin the IO interrupts among the high iops
3602          * reply queues in terms of batch count 16 when outstanding
3603          * IOs on the target device is >=8.
3604          */
3605         if (_base_sdev_nr_inflight_request(scmd->device->request_queue) >
3606             MPT3SAS_DEVICE_HIGH_IOPS_DEPTH)
3607                 return base_mod64((
3608                     atomic64_add_return(1, &ioc->high_iops_outstanding) /
3609                     MPT3SAS_HIGH_IOPS_BATCH_COUNT),
3610                     MPT3SAS_HIGH_IOPS_REPLY_QUEUES);
3611
3612         return _base_get_msix_index(ioc, scmd);
3613 }
3614
3615 /**
3616  * mpt3sas_base_get_smid - obtain a free smid from internal queue
3617  * @ioc: per adapter object
3618  * @cb_idx: callback index
3619  *
3620  * Return: smid (zero is invalid)
3621  */
3622 u16
3623 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3624 {
3625         unsigned long flags;
3626         struct request_tracker *request;
3627         u16 smid;
3628
3629         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3630         if (list_empty(&ioc->internal_free_list)) {
3631                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3632                 ioc_err(ioc, "%s: smid not available\n", __func__);
3633                 return 0;
3634         }
3635
3636         request = list_entry(ioc->internal_free_list.next,
3637             struct request_tracker, tracker_list);
3638         request->cb_idx = cb_idx;
3639         smid = request->smid;
3640         list_del(&request->tracker_list);
3641         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3642         return smid;
3643 }
3644
3645 /**
3646  * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
3647  * @ioc: per adapter object
3648  * @cb_idx: callback index
3649  * @scmd: pointer to scsi command object
3650  *
3651  * Return: smid (zero is invalid)
3652  */
3653 u16
3654 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
3655         struct scsi_cmnd *scmd)
3656 {
3657         struct scsiio_tracker *request = scsi_cmd_priv(scmd);
3658         unsigned int tag = scmd->request->tag;
3659         u16 smid;
3660
3661         smid = tag + 1;
3662         request->cb_idx = cb_idx;
3663         request->smid = smid;
3664         request->scmd = scmd;
3665         INIT_LIST_HEAD(&request->chain_list);
3666         return smid;
3667 }
3668
3669 /**
3670  * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
3671  * @ioc: per adapter object
3672  * @cb_idx: callback index
3673  *
3674  * Return: smid (zero is invalid)
3675  */
3676 u16
3677 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3678 {
3679         unsigned long flags;
3680         struct request_tracker *request;
3681         u16 smid;
3682
3683         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3684         if (list_empty(&ioc->hpr_free_list)) {
3685                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3686                 return 0;
3687         }
3688
3689         request = list_entry(ioc->hpr_free_list.next,
3690             struct request_tracker, tracker_list);
3691         request->cb_idx = cb_idx;
3692         smid = request->smid;
3693         list_del(&request->tracker_list);
3694         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3695         return smid;
3696 }
3697
3698 static void
3699 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc)
3700 {
3701         /*
3702          * See _wait_for_commands_to_complete() call with regards to this code.
3703          */
3704         if (ioc->shost_recovery && ioc->pending_io_count) {
3705                 ioc->pending_io_count = scsi_host_busy(ioc->shost);
3706                 if (ioc->pending_io_count == 0)
3707                         wake_up(&ioc->reset_wq);
3708         }
3709 }
3710
3711 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
3712                            struct scsiio_tracker *st)
3713 {
3714         if (WARN_ON(st->smid == 0))
3715                 return;
3716         st->cb_idx = 0xFF;
3717         st->direct_io = 0;
3718         st->scmd = NULL;
3719         atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);
3720         st->smid = 0;
3721 }
3722
3723 /**
3724  * mpt3sas_base_free_smid - put smid back on free_list
3725  * @ioc: per adapter object
3726  * @smid: system request message index
3727  */
3728 void
3729 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3730 {
3731         unsigned long flags;
3732         int i;
3733
3734         if (smid < ioc->hi_priority_smid) {
3735                 struct scsiio_tracker *st;
3736                 void *request;
3737
3738                 st = _get_st_from_smid(ioc, smid);
3739                 if (!st) {
3740                         _base_recovery_check(ioc);
3741                         return;
3742                 }
3743
3744                 /* Clear MPI request frame */
3745                 request = mpt3sas_base_get_msg_frame(ioc, smid);
3746                 memset(request, 0, ioc->request_sz);
3747
3748                 mpt3sas_base_clear_st(ioc, st);
3749                 _base_recovery_check(ioc);
3750                 return;
3751         }
3752
3753         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3754         if (smid < ioc->internal_smid) {
3755                 /* hi-priority */
3756                 i = smid - ioc->hi_priority_smid;
3757                 ioc->hpr_lookup[i].cb_idx = 0xFF;
3758                 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
3759         } else if (smid <= ioc->hba_queue_depth) {
3760                 /* internal queue */
3761                 i = smid - ioc->internal_smid;
3762                 ioc->internal_lookup[i].cb_idx = 0xFF;
3763                 list_add(&ioc->internal_lookup[i].tracker_list,
3764                     &ioc->internal_free_list);
3765         }
3766         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3767 }
3768
3769 /**
3770  * _base_mpi_ep_writeq - 32 bit write to MMIO
3771  * @b: data payload
3772  * @addr: address in MMIO space
3773  * @writeq_lock: spin lock
3774  *
3775  * This special handling for MPI EP to take care of 32 bit
3776  * environment where its not quarenteed to send the entire word
3777  * in one transfer.
3778  */
3779 static inline void
3780 _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
3781                                         spinlock_t *writeq_lock)
3782 {
3783         unsigned long flags;
3784
3785         spin_lock_irqsave(writeq_lock, flags);
3786         __raw_writel((u32)(b), addr);
3787         __raw_writel((u32)(b >> 32), (addr + 4));
3788         spin_unlock_irqrestore(writeq_lock, flags);
3789 }
3790
3791 /**
3792  * _base_writeq - 64 bit write to MMIO
3793  * @b: data payload
3794  * @addr: address in MMIO space
3795  * @writeq_lock: spin lock
3796  *
3797  * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
3798  * care of 32 bit environment where its not quarenteed to send the entire word
3799  * in one transfer.
3800  */
3801 #if defined(writeq) && defined(CONFIG_64BIT)
3802 static inline void
3803 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3804 {
3805         wmb();
3806         __raw_writeq(b, addr);
3807         barrier();
3808 }
3809 #else
3810 static inline void
3811 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3812 {
3813         _base_mpi_ep_writeq(b, addr, writeq_lock);
3814 }
3815 #endif
3816
3817 /**
3818  * _base_set_and_get_msix_index - get the msix index and assign to msix_io
3819  *                                variable of scsi tracker
3820  * @ioc: per adapter object
3821  * @smid: system request message index
3822  *
3823  * returns msix index.
3824  */
3825 static u8
3826 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3827 {
3828         struct scsiio_tracker *st = NULL;
3829
3830         if (smid < ioc->hi_priority_smid)
3831                 st = _get_st_from_smid(ioc, smid);
3832
3833         if (st == NULL)
3834                 return  _base_get_msix_index(ioc, NULL);
3835
3836         st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd);
3837         return st->msix_io;
3838 }
3839
3840 /**
3841  * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware
3842  * @ioc: per adapter object
3843  * @smid: system request message index
3844  * @handle: device handle
3845  */
3846 static void
3847 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc,
3848         u16 smid, u16 handle)
3849 {
3850         Mpi2RequestDescriptorUnion_t descriptor;
3851         u64 *request = (u64 *)&descriptor;
3852         void *mpi_req_iomem;
3853         __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3854
3855         _clone_sg_entries(ioc, (void *) mfp, smid);
3856         mpi_req_iomem = (void __force *)ioc->chip +
3857                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3858         _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3859                                         ioc->request_sz);
3860         descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3861         descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
3862         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3863         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3864         descriptor.SCSIIO.LMID = 0;
3865         _base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3866             &ioc->scsi_lookup_lock);
3867 }
3868
3869 /**
3870  * _base_put_smid_scsi_io - send SCSI_IO request to firmware
3871  * @ioc: per adapter object
3872  * @smid: system request message index
3873  * @handle: device handle
3874  */
3875 static void
3876 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
3877 {
3878         Mpi2RequestDescriptorUnion_t descriptor;
3879         u64 *request = (u64 *)&descriptor;
3880
3881
3882         descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3883         descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
3884         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3885         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3886         descriptor.SCSIIO.LMID = 0;
3887         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3888             &ioc->scsi_lookup_lock);
3889 }
3890
3891 /**
3892  * _base_put_smid_fast_path - send fast path request to firmware
3893  * @ioc: per adapter object
3894  * @smid: system request message index
3895  * @handle: device handle
3896  */
3897 static void
3898 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3899         u16 handle)
3900 {
3901         Mpi2RequestDescriptorUnion_t descriptor;
3902         u64 *request = (u64 *)&descriptor;
3903
3904         descriptor.SCSIIO.RequestFlags =
3905             MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3906         descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
3907         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3908         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3909         descriptor.SCSIIO.LMID = 0;
3910         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3911             &ioc->scsi_lookup_lock);
3912 }
3913
3914 /**
3915  * _base_put_smid_hi_priority - send Task Management request to firmware
3916  * @ioc: per adapter object
3917  * @smid: system request message index
3918  * @msix_task: msix_task will be same as msix of IO incase of task abort else 0.
3919  */
3920 static void
3921 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3922         u16 msix_task)
3923 {
3924         Mpi2RequestDescriptorUnion_t descriptor;
3925         void *mpi_req_iomem;
3926         u64 *request;
3927
3928         if (ioc->is_mcpu_endpoint) {
3929                 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3930
3931                 /* TBD 256 is offset within sys register. */
3932                 mpi_req_iomem = (void __force *)ioc->chip
3933                                         + MPI_FRAME_START_OFFSET
3934                                         + (smid * ioc->request_sz);
3935                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3936                                                         ioc->request_sz);
3937         }
3938
3939         request = (u64 *)&descriptor;
3940
3941         descriptor.HighPriority.RequestFlags =
3942             MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3943         descriptor.HighPriority.MSIxIndex =  msix_task;
3944         descriptor.HighPriority.SMID = cpu_to_le16(smid);
3945         descriptor.HighPriority.LMID = 0;
3946         descriptor.HighPriority.Reserved1 = 0;
3947         if (ioc->is_mcpu_endpoint)
3948                 _base_mpi_ep_writeq(*request,
3949                                 &ioc->chip->RequestDescriptorPostLow,
3950                                 &ioc->scsi_lookup_lock);
3951         else
3952                 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3953                     &ioc->scsi_lookup_lock);
3954 }
3955
3956 /**
3957  * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to
3958  *  firmware
3959  * @ioc: per adapter object
3960  * @smid: system request message index
3961  */
3962 void
3963 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3964 {
3965         Mpi2RequestDescriptorUnion_t descriptor;
3966         u64 *request = (u64 *)&descriptor;
3967
3968         descriptor.Default.RequestFlags =
3969                 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
3970         descriptor.Default.MSIxIndex =  _base_set_and_get_msix_index(ioc, smid);
3971         descriptor.Default.SMID = cpu_to_le16(smid);
3972         descriptor.Default.LMID = 0;
3973         descriptor.Default.DescriptorTypeDependent = 0;
3974         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3975             &ioc->scsi_lookup_lock);
3976 }
3977
3978 /**
3979  * _base_put_smid_default - Default, primarily used for config pages
3980  * @ioc: per adapter object
3981  * @smid: system request message index
3982  */
3983 static void
3984 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3985 {
3986         Mpi2RequestDescriptorUnion_t descriptor;
3987         void *mpi_req_iomem;
3988         u64 *request;
3989
3990         if (ioc->is_mcpu_endpoint) {
3991                 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3992
3993                 _clone_sg_entries(ioc, (void *) mfp, smid);
3994                 /* TBD 256 is offset within sys register */
3995                 mpi_req_iomem = (void __force *)ioc->chip +
3996                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3997                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3998                                                         ioc->request_sz);
3999         }
4000         request = (u64 *)&descriptor;
4001         descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
4002         descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4003         descriptor.Default.SMID = cpu_to_le16(smid);
4004         descriptor.Default.LMID = 0;
4005         descriptor.Default.DescriptorTypeDependent = 0;
4006         if (ioc->is_mcpu_endpoint)
4007                 _base_mpi_ep_writeq(*request,
4008                                 &ioc->chip->RequestDescriptorPostLow,
4009                                 &ioc->scsi_lookup_lock);
4010         else
4011                 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
4012                                 &ioc->scsi_lookup_lock);
4013 }
4014
4015 /**
4016  * _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using
4017  *   Atomic Request Descriptor
4018  * @ioc: per adapter object
4019  * @smid: system request message index
4020  * @handle: device handle, unused in this function, for function type match
4021  *
4022  * Return nothing.
4023  */
4024 static void
4025 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4026         u16 handle)
4027 {
4028         Mpi26AtomicRequestDescriptor_t descriptor;
4029         u32 *request = (u32 *)&descriptor;
4030
4031         descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
4032         descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4033         descriptor.SMID = cpu_to_le16(smid);
4034
4035         writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4036 }
4037
4038 /**
4039  * _base_put_smid_fast_path_atomic - send fast path request to firmware
4040  * using Atomic Request Descriptor
4041  * @ioc: per adapter object
4042  * @smid: system request message index
4043  * @handle: device handle, unused in this function, for function type match
4044  * Return nothing
4045  */
4046 static void
4047 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4048         u16 handle)
4049 {
4050         Mpi26AtomicRequestDescriptor_t descriptor;
4051         u32 *request = (u32 *)&descriptor;
4052
4053         descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
4054         descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4055         descriptor.SMID = cpu_to_le16(smid);
4056
4057         writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4058 }
4059
4060 /**
4061  * _base_put_smid_hi_priority_atomic - send Task Management request to
4062  * firmware using Atomic Request Descriptor
4063  * @ioc: per adapter object
4064  * @smid: system request message index
4065  * @msix_task: msix_task will be same as msix of IO incase of task abort else 0
4066  *
4067  * Return nothing.
4068  */
4069 static void
4070 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4071         u16 msix_task)
4072 {
4073         Mpi26AtomicRequestDescriptor_t descriptor;
4074         u32 *request = (u32 *)&descriptor;
4075
4076         descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
4077         descriptor.MSIxIndex = msix_task;
4078         descriptor.SMID = cpu_to_le16(smid);
4079
4080         writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4081 }
4082
4083 /**
4084  * _base_put_smid_default - Default, primarily used for config pages
4085  * use Atomic Request Descriptor
4086  * @ioc: per adapter object
4087  * @smid: system request message index
4088  *
4089  * Return nothing.
4090  */
4091 static void
4092 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4093 {
4094         Mpi26AtomicRequestDescriptor_t descriptor;
4095         u32 *request = (u32 *)&descriptor;
4096
4097         descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
4098         descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4099         descriptor.SMID = cpu_to_le16(smid);
4100
4101         writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4102 }
4103
4104 /**
4105  * _base_display_OEMs_branding - Display branding string
4106  * @ioc: per adapter object
4107  */
4108 static void
4109 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
4110 {
4111         if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
4112                 return;
4113
4114         switch (ioc->pdev->subsystem_vendor) {
4115         case PCI_VENDOR_ID_INTEL:
4116                 switch (ioc->pdev->device) {
4117                 case MPI2_MFGPAGE_DEVID_SAS2008:
4118                         switch (ioc->pdev->subsystem_device) {
4119                         case MPT2SAS_INTEL_RMS2LL080_SSDID:
4120                                 ioc_info(ioc, "%s\n",
4121                                          MPT2SAS_INTEL_RMS2LL080_BRANDING);
4122                                 break;
4123                         case MPT2SAS_INTEL_RMS2LL040_SSDID:
4124                                 ioc_info(ioc, "%s\n",
4125                                          MPT2SAS_INTEL_RMS2LL040_BRANDING);
4126                                 break;
4127                         case MPT2SAS_INTEL_SSD910_SSDID:
4128                                 ioc_info(ioc, "%s\n",
4129                                          MPT2SAS_INTEL_SSD910_BRANDING);
4130                                 break;
4131                         default:
4132                                 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4133                                          ioc->pdev->subsystem_device);
4134                                 break;
4135                         }
4136                         break;
4137                 case MPI2_MFGPAGE_DEVID_SAS2308_2:
4138                         switch (ioc->pdev->subsystem_device) {
4139                         case MPT2SAS_INTEL_RS25GB008_SSDID:
4140                                 ioc_info(ioc, "%s\n",
4141                                          MPT2SAS_INTEL_RS25GB008_BRANDING);
4142                                 break;
4143                         case MPT2SAS_INTEL_RMS25JB080_SSDID:
4144                                 ioc_info(ioc, "%s\n",
4145                                          MPT2SAS_INTEL_RMS25JB080_BRANDING);
4146                                 break;
4147                         case MPT2SAS_INTEL_RMS25JB040_SSDID:
4148                                 ioc_info(ioc, "%s\n",
4149                                          MPT2SAS_INTEL_RMS25JB040_BRANDING);
4150                                 break;
4151                         case MPT2SAS_INTEL_RMS25KB080_SSDID:
4152                                 ioc_info(ioc, "%s\n",
4153                                          MPT2SAS_INTEL_RMS25KB080_BRANDING);
4154                                 break;
4155                         case MPT2SAS_INTEL_RMS25KB040_SSDID:
4156                                 ioc_info(ioc, "%s\n",
4157                                          MPT2SAS_INTEL_RMS25KB040_BRANDING);
4158                                 break;
4159                         case MPT2SAS_INTEL_RMS25LB040_SSDID:
4160                                 ioc_info(ioc, "%s\n",
4161                                          MPT2SAS_INTEL_RMS25LB040_BRANDING);
4162                                 break;
4163                         case MPT2SAS_INTEL_RMS25LB080_SSDID:
4164                                 ioc_info(ioc, "%s\n",
4165                                          MPT2SAS_INTEL_RMS25LB080_BRANDING);
4166                                 break;
4167                         default:
4168                                 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4169                                          ioc->pdev->subsystem_device);
4170                                 break;
4171                         }
4172                         break;
4173                 case MPI25_MFGPAGE_DEVID_SAS3008:
4174                         switch (ioc->pdev->subsystem_device) {
4175                         case MPT3SAS_INTEL_RMS3JC080_SSDID:
4176                                 ioc_info(ioc, "%s\n",
4177                                          MPT3SAS_INTEL_RMS3JC080_BRANDING);
4178                                 break;
4179
4180                         case MPT3SAS_INTEL_RS3GC008_SSDID:
4181                                 ioc_info(ioc, "%s\n",
4182                                          MPT3SAS_INTEL_RS3GC008_BRANDING);
4183                                 break;
4184                         case MPT3SAS_INTEL_RS3FC044_SSDID:
4185                                 ioc_info(ioc, "%s\n",
4186                                          MPT3SAS_INTEL_RS3FC044_BRANDING);
4187                                 break;
4188                         case MPT3SAS_INTEL_RS3UC080_SSDID:
4189                                 ioc_info(ioc, "%s\n",
4190                                          MPT3SAS_INTEL_RS3UC080_BRANDING);
4191                                 break;
4192                         default:
4193                                 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4194                                          ioc->pdev->subsystem_device);
4195                                 break;
4196                         }
4197                         break;
4198                 default:
4199                         ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4200                                  ioc->pdev->subsystem_device);
4201                         break;
4202                 }
4203                 break;
4204         case PCI_VENDOR_ID_DELL:
4205                 switch (ioc->pdev->device) {
4206                 case MPI2_MFGPAGE_DEVID_SAS2008:
4207                         switch (ioc->pdev->subsystem_device) {
4208                         case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID:
4209                                 ioc_info(ioc, "%s\n",
4210                                          MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING);
4211                                 break;
4212                         case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID:
4213                                 ioc_info(ioc, "%s\n",
4214                                          MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING);
4215                                 break;
4216                         case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID:
4217                                 ioc_info(ioc, "%s\n",
4218                                          MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING);
4219                                 break;
4220                         case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID:
4221                                 ioc_info(ioc, "%s\n",
4222                                          MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING);
4223                                 break;
4224                         case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID:
4225                                 ioc_info(ioc, "%s\n",
4226                                          MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING);
4227                                 break;
4228                         case MPT2SAS_DELL_PERC_H200_SSDID:
4229                                 ioc_info(ioc, "%s\n",
4230                                          MPT2SAS_DELL_PERC_H200_BRANDING);
4231                                 break;
4232                         case MPT2SAS_DELL_6GBPS_SAS_SSDID:
4233                                 ioc_info(ioc, "%s\n",
4234                                          MPT2SAS_DELL_6GBPS_SAS_BRANDING);
4235                                 break;
4236                         default:
4237                                 ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",
4238                                          ioc->pdev->subsystem_device);
4239                                 break;
4240                         }
4241                         break;
4242                 case MPI25_MFGPAGE_DEVID_SAS3008:
4243                         switch (ioc->pdev->subsystem_device) {
4244                         case MPT3SAS_DELL_12G_HBA_SSDID:
4245                                 ioc_info(ioc, "%s\n",
4246                                          MPT3SAS_DELL_12G_HBA_BRANDING);
4247                                 break;
4248                         default:
4249                                 ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",
4250                                          ioc->pdev->subsystem_device);
4251                                 break;
4252                         }
4253                         break;
4254                 default:
4255                         ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n",
4256                                  ioc->pdev->subsystem_device);
4257                         break;
4258                 }
4259                 break;
4260         case PCI_VENDOR_ID_CISCO:
4261                 switch (ioc->pdev->device) {
4262                 case MPI25_MFGPAGE_DEVID_SAS3008:
4263                         switch (ioc->pdev->subsystem_device) {
4264                         case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
4265                                 ioc_info(ioc, "%s\n",
4266                                          MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
4267                                 break;
4268                         case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
4269                                 ioc_info(ioc, "%s\n",
4270                                          MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
4271                                 break;
4272                         case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
4273                                 ioc_info(ioc, "%s\n",
4274                                          MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
4275                                 break;
4276                         default:
4277                                 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
4278                                          ioc->pdev->subsystem_device);
4279                                 break;
4280                         }
4281                         break;
4282                 case MPI25_MFGPAGE_DEVID_SAS3108_1:
4283                         switch (ioc->pdev->subsystem_device) {
4284                         case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
4285                                 ioc_info(ioc, "%s\n",
4286                                          MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
4287                                 break;
4288                         case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
4289                                 ioc_info(ioc, "%s\n",
4290                                          MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING);
4291                                 break;
4292                         default:
4293                                 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
4294                                          ioc->pdev->subsystem_device);
4295                                 break;
4296                         }
4297                         break;
4298                 default:
4299                         ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n",
4300                                  ioc->pdev->subsystem_device);
4301                         break;
4302                 }
4303                 break;
4304         case MPT2SAS_HP_3PAR_SSVID:
4305                 switch (ioc->pdev->device) {
4306                 case MPI2_MFGPAGE_DEVID_SAS2004:
4307                         switch (ioc->pdev->subsystem_device) {
4308                         case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID:
4309                                 ioc_info(ioc, "%s\n",
4310                                          MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING);
4311                                 break;
4312                         default:
4313                                 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
4314                                          ioc->pdev->subsystem_device);
4315                                 break;
4316                         }
4317                         break;
4318                 case MPI2_MFGPAGE_DEVID_SAS2308_2:
4319                         switch (ioc->pdev->subsystem_device) {
4320                         case MPT2SAS_HP_2_4_INTERNAL_SSDID:
4321                                 ioc_info(ioc, "%s\n",
4322                                          MPT2SAS_HP_2_4_INTERNAL_BRANDING);
4323                                 break;
4324                         case MPT2SAS_HP_2_4_EXTERNAL_SSDID:
4325                                 ioc_info(ioc, "%s\n",
4326                                          MPT2SAS_HP_2_4_EXTERNAL_BRANDING);
4327                                 break;
4328                         case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID:
4329                                 ioc_info(ioc, "%s\n",
4330                                          MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING);
4331                                 break;
4332                         case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID:
4333                                 ioc_info(ioc, "%s\n",
4334                                          MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING);
4335                                 break;
4336                         default:
4337                                 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
4338                                          ioc->pdev->subsystem_device);
4339                                 break;
4340                         }
4341                         break;
4342                 default:
4343                         ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n",
4344                                  ioc->pdev->subsystem_device);
4345                         break;
4346                 }
4347         default:
4348                 break;
4349         }
4350 }
4351
4352 /**
4353  * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg
4354  *                              version from FW Image Header.
4355  * @ioc: per adapter object
4356  *
4357  * Return: 0 for success, non-zero for failure.
4358  */
4359         static int
4360 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)
4361 {
4362         Mpi2FWImageHeader_t *fw_img_hdr;
4363         Mpi26ComponentImageHeader_t *cmp_img_hdr;
4364         Mpi25FWUploadRequest_t *mpi_request;
4365         Mpi2FWUploadReply_t mpi_reply;
4366         int r = 0;
4367         u32  package_version = 0;
4368         void *fwpkg_data = NULL;
4369         dma_addr_t fwpkg_data_dma;
4370         u16 smid, ioc_status;
4371         size_t data_length;
4372
4373         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
4374
4375         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
4376                 ioc_err(ioc, "%s: internal command already in use\n", __func__);
4377                 return -EAGAIN;
4378         }
4379
4380         data_length = sizeof(Mpi2FWImageHeader_t);
4381         fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,
4382                         &fwpkg_data_dma, GFP_KERNEL);
4383         if (!fwpkg_data) {
4384                 ioc_err(ioc,
4385                     "Memory allocation for fwpkg data failed at %s:%d/%s()!\n",
4386                         __FILE__, __LINE__, __func__);
4387                 return -ENOMEM;
4388         }
4389
4390         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
4391         if (!smid) {
4392                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
4393                 r = -EAGAIN;
4394                 goto out;
4395         }
4396
4397         ioc->base_cmds.status = MPT3_CMD_PENDING;
4398         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4399         ioc->base_cmds.smid = smid;
4400         memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t));
4401         mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD;
4402         mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH;
4403         mpi_request->ImageSize = cpu_to_le32(data_length);
4404         ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,
4405                         data_length);
4406         init_completion(&ioc->base_cmds.done);
4407         ioc->put_smid_default(ioc, smid);
4408         /* Wait for 15 seconds */
4409         wait_for_completion_timeout(&ioc->base_cmds.done,
4410                         FW_IMG_HDR_READ_TIMEOUT*HZ);
4411         ioc_info(ioc, "%s: complete\n", __func__);
4412         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
4413                 ioc_err(ioc, "%s: timeout\n", __func__);
4414                 _debug_dump_mf(mpi_request,
4415                                 sizeof(Mpi25FWUploadRequest_t)/4);
4416                 r = -ETIME;
4417         } else {
4418                 memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t));
4419                 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {
4420                         memcpy(&mpi_reply, ioc->base_cmds.reply,
4421                                         sizeof(Mpi2FWUploadReply_t));
4422                         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4423                                                 MPI2_IOCSTATUS_MASK;
4424                         if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
4425                                 fw_img_hdr = (Mpi2FWImageHeader_t *)fwpkg_data;
4426                                 if (le32_to_cpu(fw_img_hdr->Signature) ==
4427                                     MPI26_IMAGE_HEADER_SIGNATURE0_MPI26) {
4428                                         cmp_img_hdr =
4429                                             (Mpi26ComponentImageHeader_t *)
4430                                             (fwpkg_data);
4431                                         package_version =
4432                                             le32_to_cpu(
4433                                             cmp_img_hdr->ApplicationSpecific);
4434                                 } else
4435                                         package_version =
4436                                             le32_to_cpu(
4437                                             fw_img_hdr->PackageVersion.Word);
4438                                 if (package_version)
4439                                         ioc_info(ioc,
4440                                         "FW Package Ver(%02d.%02d.%02d.%02d)\n",
4441                                         ((package_version) & 0xFF000000) >> 24,
4442                                         ((package_version) & 0x00FF0000) >> 16,
4443                                         ((package_version) & 0x0000FF00) >> 8,
4444                                         (package_version) & 0x000000FF);
4445                         } else {
4446                                 _debug_dump_mf(&mpi_reply,
4447                                                 sizeof(Mpi2FWUploadReply_t)/4);
4448                         }
4449                 }
4450         }
4451         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
4452 out:
4453         if (fwpkg_data)
4454                 dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data,
4455                                 fwpkg_data_dma);
4456         return r;
4457 }
4458
4459 /**
4460  * _base_display_ioc_capabilities - Disply IOC's capabilities.
4461  * @ioc: per adapter object
4462  */
4463 static void
4464 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
4465 {
4466         int i = 0;
4467         char desc[16];
4468         u32 iounit_pg1_flags;
4469         u32 bios_version;
4470
4471         bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
4472         strncpy(desc, ioc->manu_pg0.ChipName, 16);
4473         ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
4474                  desc,
4475                  (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
4476                  (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
4477                  (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
4478                  ioc->facts.FWVersion.Word & 0x000000FF,
4479                  ioc->pdev->revision,
4480                  (bios_version & 0xFF000000) >> 24,
4481                  (bios_version & 0x00FF0000) >> 16,
4482                  (bios_version & 0x0000FF00) >> 8,
4483                  bios_version & 0x000000FF);
4484
4485         _base_display_OEMs_branding(ioc);
4486
4487         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
4488                 pr_info("%sNVMe", i ? "," : "");
4489                 i++;
4490         }
4491
4492         ioc_info(ioc, "Protocol=(");
4493
4494         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
4495                 pr_cont("Initiator");
4496                 i++;
4497         }
4498
4499         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
4500                 pr_cont("%sTarget", i ? "," : "");
4501                 i++;
4502         }
4503
4504         i = 0;
4505         pr_cont("), Capabilities=(");
4506
4507         if (!ioc->hide_ir_msg) {
4508                 if (ioc->facts.IOCCapabilities &
4509                     MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
4510                         pr_cont("Raid");
4511                         i++;
4512                 }
4513         }
4514
4515         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
4516                 pr_cont("%sTLR", i ? "," : "");
4517                 i++;
4518         }
4519
4520         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
4521                 pr_cont("%sMulticast", i ? "," : "");
4522                 i++;
4523         }
4524
4525         if (ioc->facts.IOCCapabilities &
4526             MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
4527                 pr_cont("%sBIDI Target", i ? "," : "");
4528                 i++;
4529         }
4530
4531         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
4532                 pr_cont("%sEEDP", i ? "," : "");
4533                 i++;
4534         }
4535
4536         if (ioc->facts.IOCCapabilities &
4537             MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
4538                 pr_cont("%sSnapshot Buffer", i ? "," : "");
4539                 i++;
4540         }
4541
4542         if (ioc->facts.IOCCapabilities &
4543             MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
4544                 pr_cont("%sDiag Trace Buffer", i ? "," : "");
4545                 i++;
4546         }
4547
4548         if (ioc->facts.IOCCapabilities &
4549             MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
4550                 pr_cont("%sDiag Extended Buffer", i ? "," : "");
4551                 i++;
4552         }
4553
4554         if (ioc->facts.IOCCapabilities &
4555             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
4556                 pr_cont("%sTask Set Full", i ? "," : "");
4557                 i++;
4558         }
4559
4560         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4561         if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
4562                 pr_cont("%sNCQ", i ? "," : "");
4563                 i++;
4564         }
4565
4566         pr_cont(")\n");
4567 }
4568
4569 /**
4570  * mpt3sas_base_update_missing_delay - change the missing delay timers
4571  * @ioc: per adapter object
4572  * @device_missing_delay: amount of time till device is reported missing
4573  * @io_missing_delay: interval IO is returned when there is a missing device
4574  *
4575  * Passed on the command line, this function will modify the device missing
4576  * delay, as well as the io missing delay. This should be called at driver
4577  * load time.
4578  */
4579 void
4580 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
4581         u16 device_missing_delay, u8 io_missing_delay)
4582 {
4583         u16 dmd, dmd_new, dmd_orignal;
4584         u8 io_missing_delay_original;
4585         u16 sz;
4586         Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
4587         Mpi2ConfigReply_t mpi_reply;
4588         u8 num_phys = 0;
4589         u16 ioc_status;
4590
4591         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
4592         if (!num_phys)
4593                 return;
4594
4595         sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
4596             sizeof(Mpi2SasIOUnit1PhyData_t));
4597         sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
4598         if (!sas_iounit_pg1) {
4599                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
4600                         __FILE__, __LINE__, __func__);
4601                 goto out;
4602         }
4603         if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
4604             sas_iounit_pg1, sz))) {
4605                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
4606                         __FILE__, __LINE__, __func__);
4607                 goto out;
4608         }
4609         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4610             MPI2_IOCSTATUS_MASK;
4611         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4612                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
4613                         __FILE__, __LINE__, __func__);
4614                 goto out;
4615         }
4616
4617         /* device missing delay */
4618         dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
4619         if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4620                 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4621         else
4622                 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4623         dmd_orignal = dmd;
4624         if (device_missing_delay > 0x7F) {
4625                 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
4626                     device_missing_delay;
4627                 dmd = dmd / 16;
4628                 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
4629         } else
4630                 dmd = device_missing_delay;
4631         sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
4632
4633         /* io missing delay */
4634         io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
4635         sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
4636
4637         if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
4638             sz)) {
4639                 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4640                         dmd_new = (dmd &
4641                             MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4642                 else
4643                         dmd_new =
4644                     dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4645                 ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n",
4646                          dmd_orignal, dmd_new);
4647                 ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n",
4648                          io_missing_delay_original,
4649                          io_missing_delay);
4650                 ioc->device_missing_delay = dmd_new;
4651                 ioc->io_missing_delay = io_missing_delay;
4652         }
4653
4654 out:
4655         kfree(sas_iounit_pg1);
4656 }
4657
4658 /**
4659  * _base_update_ioc_page1_inlinewith_perf_mode - Update IOC Page1 fields
4660  *    according to performance mode.
4661  * @ioc : per adapter object
4662  *
4663  * Return nothing.
4664  */
4665 static void
4666 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc)
4667 {
4668         Mpi2IOCPage1_t ioc_pg1;
4669         Mpi2ConfigReply_t mpi_reply;
4670
4671         mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy);
4672         memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t));
4673
4674         switch (perf_mode) {
4675         case MPT_PERF_MODE_DEFAULT:
4676         case MPT_PERF_MODE_BALANCED:
4677                 if (ioc->high_iops_queues) {
4678                         ioc_info(ioc,
4679                                 "Enable interrupt coalescing only for first\t"
4680                                 "%d reply queues\n",
4681                                 MPT3SAS_HIGH_IOPS_REPLY_QUEUES);
4682                         /*
4683                          * If 31st bit is zero then interrupt coalescing is
4684                          * enabled for all reply descriptor post queues.
4685                          * If 31st bit is set to one then user can
4686                          * enable/disable interrupt coalescing on per reply
4687                          * descriptor post queue group(8) basis. So to enable
4688                          * interrupt coalescing only on first reply descriptor
4689                          * post queue group 31st bit and zero th bit is enabled.
4690                          */
4691                         ioc_pg1.ProductSpecific = cpu_to_le32(0x80000000 |
4692                             ((1 << MPT3SAS_HIGH_IOPS_REPLY_QUEUES/8) - 1));
4693                         mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4694                         ioc_info(ioc, "performance mode: balanced\n");
4695                         return;
4696                 }
4697                 fallthrough;
4698         case MPT_PERF_MODE_LATENCY:
4699                 /*
4700                  * Enable interrupt coalescing on all reply queues
4701                  * with timeout value 0xA
4702                  */
4703                 ioc_pg1.CoalescingTimeout = cpu_to_le32(0xa);
4704                 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING);
4705                 ioc_pg1.ProductSpecific = 0;
4706                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4707                 ioc_info(ioc, "performance mode: latency\n");
4708                 break;
4709         case MPT_PERF_MODE_IOPS:
4710                 /*
4711                  * Enable interrupt coalescing on all reply queues.
4712                  */
4713                 ioc_info(ioc,
4714                     "performance mode: iops with coalescing timeout: 0x%x\n",
4715                     le32_to_cpu(ioc_pg1.CoalescingTimeout));
4716                 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING);
4717                 ioc_pg1.ProductSpecific = 0;
4718                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4719                 break;
4720         }
4721 }
4722
4723 /**
4724  * _base_static_config_pages - static start of day config pages
4725  * @ioc: per adapter object
4726  */
4727 static void
4728 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
4729 {
4730         Mpi2ConfigReply_t mpi_reply;
4731         u32 iounit_pg1_flags;
4732
4733         ioc->nvme_abort_timeout = 30;
4734         mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
4735         if (ioc->ir_firmware)
4736                 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
4737                     &ioc->manu_pg10);
4738
4739         /*
4740          * Ensure correct T10 PI operation if vendor left EEDPTagMode
4741          * flag unset in NVDATA.
4742          */
4743         mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11);
4744         if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) {
4745                 pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
4746                     ioc->name);
4747                 ioc->manu_pg11.EEDPTagMode &= ~0x3;
4748                 ioc->manu_pg11.EEDPTagMode |= 0x1;
4749                 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
4750                     &ioc->manu_pg11);
4751         }
4752         if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)
4753                 ioc->tm_custom_handling = 1;
4754         else {
4755                 ioc->tm_custom_handling = 0;
4756                 if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)
4757                         ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;
4758                 else if (ioc->manu_pg11.NVMeAbortTO >
4759                                         NVME_TASK_ABORT_MAX_TIMEOUT)
4760                         ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;
4761                 else
4762                         ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;
4763         }
4764
4765         mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
4766         mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
4767         mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
4768         mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
4769         mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
4770         mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
4771         _base_display_ioc_capabilities(ioc);
4772
4773         /*
4774          * Enable task_set_full handling in iounit_pg1 when the
4775          * facts capabilities indicate that its supported.
4776          */
4777         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4778         if ((ioc->facts.IOCCapabilities &
4779             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
4780                 iounit_pg1_flags &=
4781                     ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4782         else
4783                 iounit_pg1_flags |=
4784                     MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4785         ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
4786         mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
4787
4788         if (ioc->iounit_pg8.NumSensors)
4789                 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
4790         if (ioc->is_aero_ioc)
4791                 _base_update_ioc_page1_inlinewith_perf_mode(ioc);
4792 }
4793
4794 /**
4795  * mpt3sas_free_enclosure_list - release memory
4796  * @ioc: per adapter object
4797  *
4798  * Free memory allocated during encloure add.
4799  */
4800 void
4801 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)
4802 {
4803         struct _enclosure_node *enclosure_dev, *enclosure_dev_next;
4804
4805         /* Free enclosure list */
4806         list_for_each_entry_safe(enclosure_dev,
4807                         enclosure_dev_next, &ioc->enclosure_list, list) {
4808                 list_del(&enclosure_dev->list);
4809                 kfree(enclosure_dev);
4810         }
4811 }
4812
4813 /**
4814  * _base_release_memory_pools - release memory
4815  * @ioc: per adapter object
4816  *
4817  * Free memory allocated from _base_allocate_memory_pools.
4818  */
4819 static void
4820 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
4821 {
4822         int i = 0;
4823         int j = 0;
4824         int dma_alloc_count = 0;
4825         struct chain_tracker *ct;
4826         int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
4827
4828         dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
4829
4830         if (ioc->request) {
4831                 dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz,
4832                     ioc->request,  ioc->request_dma);
4833                 dexitprintk(ioc,
4834                             ioc_info(ioc, "request_pool(0x%p): free\n",
4835                                      ioc->request));
4836                 ioc->request = NULL;
4837         }
4838
4839         if (ioc->sense) {
4840                 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
4841                 dma_pool_destroy(ioc->sense_dma_pool);
4842                 dexitprintk(ioc,
4843                             ioc_info(ioc, "sense_pool(0x%p): free\n",
4844                                      ioc->sense));
4845                 ioc->sense = NULL;
4846         }
4847
4848         if (ioc->reply) {
4849                 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
4850                 dma_pool_destroy(ioc->reply_dma_pool);
4851                 dexitprintk(ioc,
4852                             ioc_info(ioc, "reply_pool(0x%p): free\n",
4853                                      ioc->reply));
4854                 ioc->reply = NULL;
4855         }
4856
4857         if (ioc->reply_free) {
4858                 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
4859                     ioc->reply_free_dma);
4860                 dma_pool_destroy(ioc->reply_free_dma_pool);
4861                 dexitprintk(ioc,
4862                             ioc_info(ioc, "reply_free_pool(0x%p): free\n",
4863                                      ioc->reply_free));
4864                 ioc->reply_free = NULL;
4865         }
4866
4867         if (ioc->reply_post) {
4868                 dma_alloc_count = DIV_ROUND_UP(count,
4869                                 RDPQ_MAX_INDEX_IN_ONE_CHUNK);
4870                 for (i = 0; i < count; i++) {
4871                         if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0
4872                             && dma_alloc_count) {
4873                                 if (ioc->reply_post[i].reply_post_free) {
4874                                         dma_pool_free(
4875                                             ioc->reply_post_free_dma_pool,
4876                                             ioc->reply_post[i].reply_post_free,
4877                                         ioc->reply_post[i].reply_post_free_dma);
4878                                         dexitprintk(ioc, ioc_info(ioc,
4879                                            "reply_post_free_pool(0x%p): free\n",
4880                                            ioc->reply_post[i].reply_post_free));
4881                                         ioc->reply_post[i].reply_post_free =
4882                                                                         NULL;
4883                                 }
4884                                 --dma_alloc_count;
4885                         }
4886                 }
4887                 dma_pool_destroy(ioc->reply_post_free_dma_pool);
4888                 if (ioc->reply_post_free_array &&
4889                         ioc->rdpq_array_enable) {
4890                         dma_pool_free(ioc->reply_post_free_array_dma_pool,
4891                             ioc->reply_post_free_array,
4892                             ioc->reply_post_free_array_dma);
4893                         ioc->reply_post_free_array = NULL;
4894                 }
4895                 dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
4896                 kfree(ioc->reply_post);
4897         }
4898
4899         if (ioc->pcie_sgl_dma_pool) {
4900                 for (i = 0; i < ioc->scsiio_depth; i++) {
4901                         dma_pool_free(ioc->pcie_sgl_dma_pool,
4902                                         ioc->pcie_sg_lookup[i].pcie_sgl,
4903                                         ioc->pcie_sg_lookup[i].pcie_sgl_dma);
4904                         ioc->pcie_sg_lookup[i].pcie_sgl = NULL;
4905                 }
4906                 dma_pool_destroy(ioc->pcie_sgl_dma_pool);
4907         }
4908         if (ioc->config_page) {
4909                 dexitprintk(ioc,
4910                             ioc_info(ioc, "config_page(0x%p): free\n",
4911                                      ioc->config_page));
4912                 dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz,
4913                     ioc->config_page, ioc->config_page_dma);
4914         }
4915
4916         kfree(ioc->hpr_lookup);
4917         ioc->hpr_lookup = NULL;
4918         kfree(ioc->internal_lookup);
4919         ioc->internal_lookup = NULL;
4920         if (ioc->chain_lookup) {
4921                 for (i = 0; i < ioc->scsiio_depth; i++) {
4922                         for (j = ioc->chains_per_prp_buffer;
4923                             j < ioc->chains_needed_per_io; j++) {
4924                                 ct = &ioc->chain_lookup[i].chains_per_smid[j];
4925                                 if (ct && ct->chain_buffer)
4926                                         dma_pool_free(ioc->chain_dma_pool,
4927                                                 ct->chain_buffer,
4928                                                 ct->chain_buffer_dma);
4929                         }
4930                         kfree(ioc->chain_lookup[i].chains_per_smid);
4931                 }
4932                 dma_pool_destroy(ioc->chain_dma_pool);
4933                 kfree(ioc->chain_lookup);
4934                 ioc->chain_lookup = NULL;
4935         }
4936 }
4937
4938 /**
4939  * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are
4940  *      having same upper 32bits in their base memory address.
4941  * @reply_pool_start_address: Base address of a reply queue set
4942  * @pool_sz: Size of single Reply Descriptor Post Queues pool size
4943  *
4944  * Return: 1 if reply queues in a set have a same upper 32bits in their base
4945  * memory address, else 0.
4946  */
4947
4948 static int
4949 mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz)
4950 {
4951         long reply_pool_end_address;
4952
4953         reply_pool_end_address = reply_pool_start_address + pool_sz;
4954
4955         if (upper_32_bits(reply_pool_start_address) ==
4956                 upper_32_bits(reply_pool_end_address))
4957                 return 1;
4958         else
4959                 return 0;
4960 }
4961
4962 /**
4963  * _base_reduce_hba_queue_depth- Retry with reduced queue depth
4964  * @ioc: Adapter object
4965  *
4966  * Return: 0 for success, non-zero for failure.
4967  **/
4968 static inline int
4969 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc)
4970 {
4971         int reduce_sz = 64;
4972
4973         if ((ioc->hba_queue_depth - reduce_sz) >
4974             (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) {
4975                 ioc->hba_queue_depth -= reduce_sz;
4976                 return 0;
4977         } else
4978                 return -ENOMEM;
4979 }
4980
4981 /**
4982  * _base_allocate_pcie_sgl_pool - Allocating DMA'able memory
4983  *                      for pcie sgl pools.
4984  * @ioc: Adapter object
4985  * @sz: DMA Pool size
4986  * @ct: Chain tracker
4987  * Return: 0 for success, non-zero for failure.
4988  */
4989
4990 static int
4991 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
4992 {
4993         int i = 0, j = 0;
4994         struct chain_tracker *ct;
4995
4996         ioc->pcie_sgl_dma_pool =
4997             dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz,
4998             ioc->page_size, 0);
4999         if (!ioc->pcie_sgl_dma_pool) {
5000                 ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n");
5001                 return -ENOMEM;
5002         }
5003
5004         ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
5005         ioc->chains_per_prp_buffer =
5006             min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io);
5007         for (i = 0; i < ioc->scsiio_depth; i++) {
5008                 ioc->pcie_sg_lookup[i].pcie_sgl =
5009                     dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL,
5010                     &ioc->pcie_sg_lookup[i].pcie_sgl_dma);
5011                 if (!ioc->pcie_sg_lookup[i].pcie_sgl) {
5012                         ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n");
5013                         return -EAGAIN;
5014                 }
5015
5016                 if (!mpt3sas_check_same_4gb_region(
5017                     (long)ioc->pcie_sg_lookup[i].pcie_sgl, sz)) {
5018                         ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n",
5019                             ioc->pcie_sg_lookup[i].pcie_sgl,
5020                             (unsigned long long)
5021                             ioc->pcie_sg_lookup[i].pcie_sgl_dma);
5022                         ioc->use_32bit_dma = true;
5023                         return -EAGAIN;
5024                 }
5025
5026                 for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
5027                         ct = &ioc->chain_lookup[i].chains_per_smid[j];
5028                         ct->chain_buffer =
5029                             ioc->pcie_sg_lookup[i].pcie_sgl +
5030                             (j * ioc->chain_segment_sz);
5031                         ct->chain_buffer_dma =
5032                             ioc->pcie_sg_lookup[i].pcie_sgl_dma +
5033                             (j * ioc->chain_segment_sz);
5034                 }
5035         }
5036         dinitprintk(ioc, ioc_info(ioc,
5037             "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n",
5038             ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
5039         dinitprintk(ioc, ioc_info(ioc,
5040             "Number of chains can fit in a PRP page(%d)\n",
5041             ioc->chains_per_prp_buffer));
5042         return 0;
5043 }
5044
5045 /**
5046  * base_alloc_rdpq_dma_pool - Allocating DMA'able memory
5047  *                     for reply queues.
5048  * @ioc: per adapter object
5049  * @sz: DMA Pool size
5050  * Return: 0 for success, non-zero for failure.
5051  */
5052 static int
5053 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)
5054 {
5055         int i = 0;
5056         u32 dma_alloc_count = 0;
5057         int reply_post_free_sz = ioc->reply_post_queue_depth *
5058                 sizeof(Mpi2DefaultReplyDescriptor_t);
5059         int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
5060
5061         ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct),
5062                         GFP_KERNEL);
5063         if (!ioc->reply_post)
5064                 return -ENOMEM;
5065         /*
5066          *  For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
5067          *  VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should
5068          *  be within 4GB boundary i.e reply queues in a set must have same
5069          *  upper 32-bits in their memory address. so here driver is allocating
5070          *  the DMA'able memory for reply queues according.
5071          *  Driver uses limitation of
5072          *  VENTURA_SERIES to manage INVADER_SERIES as well.
5073          */
5074         dma_alloc_count = DIV_ROUND_UP(count,
5075                                 RDPQ_MAX_INDEX_IN_ONE_CHUNK);
5076         ioc->reply_post_free_dma_pool =
5077                 dma_pool_create("reply_post_free pool",
5078                     &ioc->pdev->dev, sz, 16, 0);
5079         if (!ioc->reply_post_free_dma_pool)
5080                 return -ENOMEM;
5081         for (i = 0; i < count; i++) {
5082                 if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) {
5083                         ioc->reply_post[i].reply_post_free =
5084                             dma_pool_zalloc(ioc->reply_post_free_dma_pool,
5085                                 GFP_KERNEL,
5086                                 &ioc->reply_post[i].reply_post_free_dma);
5087                         if (!ioc->reply_post[i].reply_post_free)
5088                                 return -ENOMEM;
5089                         /*
5090                          * Each set of RDPQ pool must satisfy 4gb boundary
5091                          * restriction.
5092                          * 1) Check if allocated resources for RDPQ pool are in
5093                          *      the same 4GB range.
5094                          * 2) If #1 is true, continue with 64 bit DMA.
5095                          * 3) If #1 is false, return 1. which means free all the
5096                          * resources and set DMA mask to 32 and allocate.
5097                          */
5098                         if (!mpt3sas_check_same_4gb_region(
5099                                 (long)ioc->reply_post[i].reply_post_free, sz)) {
5100                                 dinitprintk(ioc,
5101                                     ioc_err(ioc, "bad Replypost free pool(0x%p)"
5102                                     "reply_post_free_dma = (0x%llx)\n",
5103                                     ioc->reply_post[i].reply_post_free,
5104                                     (unsigned long long)
5105                                     ioc->reply_post[i].reply_post_free_dma));
5106                                 return -EAGAIN;
5107                         }
5108                         dma_alloc_count--;
5109
5110                 } else {
5111                         ioc->reply_post[i].reply_post_free =
5112                             (Mpi2ReplyDescriptorsUnion_t *)
5113                             ((long)ioc->reply_post[i-1].reply_post_free
5114                             + reply_post_free_sz);
5115                         ioc->reply_post[i].reply_post_free_dma =
5116                             (dma_addr_t)
5117                             (ioc->reply_post[i-1].reply_post_free_dma +
5118                             reply_post_free_sz);
5119                 }
5120         }
5121         return 0;
5122 }
5123
5124 /**
5125  * _base_allocate_memory_pools - allocate start of day memory pools
5126  * @ioc: per adapter object
5127  *
5128  * Return: 0 success, anything else error.
5129  */
5130 static int
5131 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
5132 {
5133         struct mpt3sas_facts *facts;
5134         u16 max_sge_elements;
5135         u16 chains_needed_per_io;
5136         u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
5137         u32 retry_sz;
5138         u32 rdpq_sz = 0;
5139         u16 max_request_credit, nvme_blocks_needed;
5140         unsigned short sg_tablesize;
5141         u16 sge_size;
5142         int i, j;
5143         int ret = 0, rc = 0;
5144         struct chain_tracker *ct;
5145
5146         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
5147
5148
5149         retry_sz = 0;
5150         facts = &ioc->facts;
5151
5152         /* command line tunables for max sgl entries */
5153         if (max_sgl_entries != -1)
5154                 sg_tablesize = max_sgl_entries;
5155         else {
5156                 if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
5157                         sg_tablesize = MPT2SAS_SG_DEPTH;
5158                 else
5159                         sg_tablesize = MPT3SAS_SG_DEPTH;
5160         }
5161
5162         /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */
5163         if (reset_devices)
5164                 sg_tablesize = min_t(unsigned short, sg_tablesize,
5165                    MPT_KDUMP_MIN_PHYS_SEGMENTS);
5166
5167         if (ioc->is_mcpu_endpoint)
5168                 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
5169         else {
5170                 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
5171                         sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
5172                 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
5173                         sg_tablesize = min_t(unsigned short, sg_tablesize,
5174                                         SG_MAX_SEGMENTS);
5175                         ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n",
5176                                  sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
5177                 }
5178                 ioc->shost->sg_tablesize = sg_tablesize;
5179         }
5180
5181         ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),
5182                 (facts->RequestCredit / 4));
5183         if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {
5184                 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT +
5185                                 INTERNAL_SCSIIO_CMDS_COUNT)) {
5186                         ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n",
5187                                 facts->RequestCredit);
5188                         return -ENOMEM;
5189                 }
5190                 ioc->internal_depth = 10;
5191         }
5192
5193         ioc->hi_priority_depth = ioc->internal_depth - (5);
5194         /* command line tunables  for max controller queue depth */
5195         if (max_queue_depth != -1 && max_queue_depth != 0) {
5196                 max_request_credit = min_t(u16, max_queue_depth +
5197                         ioc->internal_depth, facts->RequestCredit);
5198                 if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
5199                         max_request_credit =  MAX_HBA_QUEUE_DEPTH;
5200         } else if (reset_devices)
5201                 max_request_credit = min_t(u16, facts->RequestCredit,
5202                     (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));
5203         else
5204                 max_request_credit = min_t(u16, facts->RequestCredit,
5205                     MAX_HBA_QUEUE_DEPTH);
5206
5207         /* Firmware maintains additional facts->HighPriorityCredit number of
5208          * credits for HiPriprity Request messages, so hba queue depth will be
5209          * sum of max_request_credit and high priority queue depth.
5210          */
5211         ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;
5212
5213         /* request frame size */
5214         ioc->request_sz = facts->IOCRequestFrameSize * 4;
5215
5216         /* reply frame size */
5217         ioc->reply_sz = facts->ReplyFrameSize * 4;
5218
5219         /* chain segment size */
5220         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
5221                 if (facts->IOCMaxChainSegmentSize)
5222                         ioc->chain_segment_sz =
5223                                         facts->IOCMaxChainSegmentSize *
5224                                         MAX_CHAIN_ELEMT_SZ;
5225                 else
5226                 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */
5227                         ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *
5228                                                     MAX_CHAIN_ELEMT_SZ;
5229         } else
5230                 ioc->chain_segment_sz = ioc->request_sz;
5231
5232         /* calculate the max scatter element size */
5233         sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
5234
5235  retry_allocation:
5236         total_sz = 0;
5237         /* calculate number of sg elements left over in the 1st frame */
5238         max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
5239             sizeof(Mpi2SGEIOUnion_t)) + sge_size);
5240         ioc->max_sges_in_main_message = max_sge_elements/sge_size;
5241
5242         /* now do the same for a chain buffer */
5243         max_sge_elements = ioc->chain_segment_sz - sge_size;
5244         ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
5245
5246         /*
5247          *  MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
5248          */
5249         chains_needed_per_io = ((ioc->shost->sg_tablesize -
5250            ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
5251             + 1;
5252         if (chains_needed_per_io > facts->MaxChainDepth) {
5253                 chains_needed_per_io = facts->MaxChainDepth;
5254                 ioc->shost->sg_tablesize = min_t(u16,
5255                 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
5256                 * chains_needed_per_io), ioc->shost->sg_tablesize);
5257         }
5258         ioc->chains_needed_per_io = chains_needed_per_io;
5259
5260         /* reply free queue sizing - taking into account for 64 FW events */
5261         ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
5262
5263         /* mCPU manage single counters for simplicity */
5264         if (ioc->is_mcpu_endpoint)
5265                 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;
5266         else {
5267                 /* calculate reply descriptor post queue depth */
5268                 ioc->reply_post_queue_depth = ioc->hba_queue_depth +
5269                         ioc->reply_free_queue_depth +  1;
5270                 /* align the reply post queue on the next 16 count boundary */
5271                 if (ioc->reply_post_queue_depth % 16)
5272                         ioc->reply_post_queue_depth += 16 -
5273                                 (ioc->reply_post_queue_depth % 16);
5274         }
5275
5276         if (ioc->reply_post_queue_depth >
5277             facts->MaxReplyDescriptorPostQueueDepth) {
5278                 ioc->reply_post_queue_depth =
5279                                 facts->MaxReplyDescriptorPostQueueDepth -
5280                     (facts->MaxReplyDescriptorPostQueueDepth % 16);
5281                 ioc->hba_queue_depth =
5282                                 ((ioc->reply_post_queue_depth - 64) / 2) - 1;
5283                 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
5284         }
5285
5286         ioc_info(ioc,
5287             "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), "
5288             "sge_per_io(%d), chains_per_io(%d)\n",
5289             ioc->max_sges_in_main_message,
5290             ioc->max_sges_in_chain_message,
5291             ioc->shost->sg_tablesize,
5292             ioc->chains_needed_per_io);
5293
5294         /* reply post queue, 16 byte align */
5295         reply_post_free_sz = ioc->reply_post_queue_depth *
5296             sizeof(Mpi2DefaultReplyDescriptor_t);
5297         rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
5298         if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
5299                 rdpq_sz = reply_post_free_sz * ioc->reply_queue_count;
5300         ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz);
5301         if (ret == -EAGAIN) {
5302                 /*
5303                  * Free allocated bad RDPQ memory pools.
5304                  * Change dma coherent mask to 32 bit and reallocate RDPQ
5305                  */
5306                 _base_release_memory_pools(ioc);
5307                 ioc->use_32bit_dma = true;
5308                 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {
5309                         ioc_err(ioc,
5310                             "32 DMA mask failed %s\n", pci_name(ioc->pdev));
5311                         return -ENODEV;
5312                 }
5313                 if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz))
5314                         return -ENOMEM;
5315         } else if (ret == -ENOMEM)
5316                 return -ENOMEM;
5317         total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 :
5318             DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK));
5319         ioc->scsiio_depth = ioc->hba_queue_depth -
5320             ioc->hi_priority_depth - ioc->internal_depth;
5321
5322         /* set the scsi host can_queue depth
5323          * with some internal commands that could be outstanding
5324          */
5325         ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;
5326         dinitprintk(ioc,
5327                     ioc_info(ioc, "scsi host: can_queue depth (%d)\n",
5328                              ioc->shost->can_queue));
5329
5330         /* contiguous pool for request and chains, 16 byte align, one extra "
5331          * "frame for smid=0
5332          */
5333         ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
5334         sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
5335
5336         /* hi-priority queue */
5337         sz += (ioc->hi_priority_depth * ioc->request_sz);
5338
5339         /* internal queue */
5340         sz += (ioc->internal_depth * ioc->request_sz);
5341
5342         ioc->request_dma_sz = sz;
5343         ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz,
5344                         &ioc->request_dma, GFP_KERNEL);
5345         if (!ioc->request) {
5346                 ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kB)\n",
5347                         ioc->hba_queue_depth, ioc->chains_needed_per_io,
5348                         ioc->request_sz, sz / 1024);
5349                 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
5350                         goto out;
5351                 retry_sz = 64;
5352                 ioc->hba_queue_depth -= retry_sz;
5353                 _base_release_memory_pools(ioc);
5354                 goto retry_allocation;
5355         }
5356
5357         if (retry_sz)
5358                 ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kb)\n",
5359                         ioc->hba_queue_depth, ioc->chains_needed_per_io,
5360                         ioc->request_sz, sz / 1024);
5361
5362         /* hi-priority queue */
5363         ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
5364             ioc->request_sz);
5365         ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
5366             ioc->request_sz);
5367
5368         /* internal queue */
5369         ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
5370             ioc->request_sz);
5371         ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
5372             ioc->request_sz);
5373
5374         ioc_info(ioc,
5375             "request pool(0x%p) - dma(0x%llx): "
5376             "depth(%d), frame_size(%d), pool_size(%d kB)\n",
5377             ioc->request, (unsigned long long) ioc->request_dma,
5378             ioc->hba_queue_depth, ioc->request_sz,
5379             (ioc->hba_queue_depth * ioc->request_sz) / 1024);
5380
5381         total_sz += sz;
5382
5383         dinitprintk(ioc,
5384                     ioc_info(ioc, "scsiio(0x%p): depth(%d)\n",
5385                              ioc->request, ioc->scsiio_depth));
5386
5387         ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
5388         sz = ioc->scsiio_depth * sizeof(struct chain_lookup);
5389         ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
5390         if (!ioc->chain_lookup) {
5391                 ioc_err(ioc, "chain_lookup: __get_free_pages failed\n");
5392                 goto out;
5393         }
5394
5395         sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);
5396         for (i = 0; i < ioc->scsiio_depth; i++) {
5397                 ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);
5398                 if (!ioc->chain_lookup[i].chains_per_smid) {
5399                         ioc_err(ioc, "chain_lookup: kzalloc failed\n");
5400                         goto out;
5401                 }
5402         }
5403
5404         /* initialize hi-priority queue smid's */
5405         ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
5406             sizeof(struct request_tracker), GFP_KERNEL);
5407         if (!ioc->hpr_lookup) {
5408                 ioc_err(ioc, "hpr_lookup: kcalloc failed\n");
5409                 goto out;
5410         }
5411         ioc->hi_priority_smid = ioc->scsiio_depth + 1;
5412         dinitprintk(ioc,
5413                     ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n",
5414                              ioc->hi_priority,
5415                              ioc->hi_priority_depth, ioc->hi_priority_smid));
5416
5417         /* initialize internal queue smid's */
5418         ioc->internal_lookup = kcalloc(ioc->internal_depth,
5419             sizeof(struct request_tracker), GFP_KERNEL);
5420         if (!ioc->internal_lookup) {
5421                 ioc_err(ioc, "internal_lookup: kcalloc failed\n");
5422                 goto out;
5423         }
5424         ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
5425         dinitprintk(ioc,
5426                     ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n",
5427                              ioc->internal,
5428                              ioc->internal_depth, ioc->internal_smid));
5429         /*
5430          * The number of NVMe page sized blocks needed is:
5431          *     (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1
5432          * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry
5433          * that is placed in the main message frame.  8 is the size of each PRP
5434          * entry or PRP list pointer entry.  8 is subtracted from page_size
5435          * because of the PRP list pointer entry at the end of a page, so this
5436          * is not counted as a PRP entry.  The 1 added page is a round up.
5437          *
5438          * To avoid allocation failures due to the amount of memory that could
5439          * be required for NVMe PRP's, only each set of NVMe blocks will be
5440          * contiguous, so a new set is allocated for each possible I/O.
5441          */
5442
5443         ioc->chains_per_prp_buffer = 0;
5444         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
5445                 nvme_blocks_needed =
5446                         (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
5447                 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);
5448                 nvme_blocks_needed++;
5449
5450                 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;
5451                 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);
5452                 if (!ioc->pcie_sg_lookup) {
5453                         ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n");
5454                         goto out;
5455                 }
5456                 sz = nvme_blocks_needed * ioc->page_size;
5457                 rc = _base_allocate_pcie_sgl_pool(ioc, sz);
5458                 if (rc == -ENOMEM)
5459                         return -ENOMEM;
5460                 else if (rc == -EAGAIN)
5461                         goto try_32bit_dma;
5462                 total_sz += sz * ioc->scsiio_depth;
5463         }
5464
5465         ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
5466             ioc->chain_segment_sz, 16, 0);
5467         if (!ioc->chain_dma_pool) {
5468                 ioc_err(ioc, "chain_dma_pool: dma_pool_create failed\n");
5469                 goto out;
5470         }
5471         for (i = 0; i < ioc->scsiio_depth; i++) {
5472                 for (j = ioc->chains_per_prp_buffer;
5473                                 j < ioc->chains_needed_per_io; j++) {
5474                         ct = &ioc->chain_lookup[i].chains_per_smid[j];
5475                         ct->chain_buffer = dma_pool_alloc(
5476                                         ioc->chain_dma_pool, GFP_KERNEL,
5477                                         &ct->chain_buffer_dma);
5478                         if (!ct->chain_buffer) {
5479                                 ioc_err(ioc, "chain_lookup: pci_pool_alloc failed\n");
5480                                 goto out;
5481                         }
5482                 }
5483                 total_sz += ioc->chain_segment_sz;
5484         }
5485
5486         dinitprintk(ioc,
5487                     ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
5488                              ioc->chain_depth, ioc->chain_segment_sz,
5489                              (ioc->chain_depth * ioc->chain_segment_sz) / 1024));
5490
5491         /* sense buffers, 4 byte align */
5492         sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
5493         ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
5494                                               4, 0);
5495         if (!ioc->sense_dma_pool) {
5496                 ioc_err(ioc, "sense pool: dma_pool_create failed\n");
5497                 goto out;
5498         }
5499         ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
5500             &ioc->sense_dma);
5501         if (!ioc->sense) {
5502                 ioc_err(ioc, "sense pool: dma_pool_alloc failed\n");
5503                 goto out;
5504         }
5505         /* sense buffer requires to be in same 4 gb region.
5506          * Below function will check the same.
5507          * In case of failure, new pci pool will be created with updated
5508          * alignment. Older allocation and pool will be destroyed.
5509          * Alignment will be used such a way that next allocation if
5510          * success, will always meet same 4gb region requirement.
5511          * Actual requirement is not alignment, but we need start and end of
5512          * DMA address must have same upper 32 bit address.
5513          */
5514         if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) {
5515                 //Release Sense pool & Reallocate
5516                 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
5517                 dma_pool_destroy(ioc->sense_dma_pool);
5518                 ioc->sense = NULL;
5519
5520                 ioc->sense_dma_pool =
5521                         dma_pool_create("sense pool", &ioc->pdev->dev, sz,
5522                                                 roundup_pow_of_two(sz), 0);
5523                 if (!ioc->sense_dma_pool) {
5524                         ioc_err(ioc, "sense pool: pci_pool_create failed\n");
5525                         goto out;
5526                 }
5527                 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
5528                                 &ioc->sense_dma);
5529                 if (!ioc->sense) {
5530                         ioc_err(ioc, "sense pool: pci_pool_alloc failed\n");
5531                         goto out;
5532                 }
5533         }
5534         ioc_info(ioc,
5535             "sense pool(0x%p)- dma(0x%llx): depth(%d),"
5536             "element_size(%d), pool_size(%d kB)\n",
5537             ioc->sense, (unsigned long long)ioc->sense_dma, ioc->scsiio_depth,
5538             SCSI_SENSE_BUFFERSIZE, sz / 1024);
5539
5540         total_sz += sz;
5541
5542         /* reply pool, 4 byte align */
5543         sz = ioc->reply_free_queue_depth * ioc->reply_sz;
5544         ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz,
5545                                               4, 0);
5546         if (!ioc->reply_dma_pool) {
5547                 ioc_err(ioc, "reply pool: dma_pool_create failed\n");
5548                 goto out;
5549         }
5550         ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
5551             &ioc->reply_dma);
5552         if (!ioc->reply) {
5553                 ioc_err(ioc, "reply pool: dma_pool_alloc failed\n");
5554                 goto out;
5555         }
5556         ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
5557         ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
5558         dinitprintk(ioc,
5559                     ioc_info(ioc, "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
5560                              ioc->reply, ioc->reply_free_queue_depth,
5561                              ioc->reply_sz, sz / 1024));
5562         dinitprintk(ioc,
5563                     ioc_info(ioc, "reply_dma(0x%llx)\n",
5564                              (unsigned long long)ioc->reply_dma));
5565         total_sz += sz;
5566
5567         /* reply free queue, 16 byte align */
5568         sz = ioc->reply_free_queue_depth * 4;
5569         ioc->reply_free_dma_pool = dma_pool_create("reply_free pool",
5570             &ioc->pdev->dev, sz, 16, 0);
5571         if (!ioc->reply_free_dma_pool) {
5572                 ioc_err(ioc, "reply_free pool: dma_pool_create failed\n");
5573                 goto out;
5574         }
5575         ioc->reply_free = dma_pool_zalloc(ioc->reply_free_dma_pool, GFP_KERNEL,
5576             &ioc->reply_free_dma);
5577         if (!ioc->reply_free) {
5578                 ioc_err(ioc, "reply_free pool: dma_pool_alloc failed\n");
5579                 goto out;
5580         }
5581         dinitprintk(ioc,
5582                     ioc_info(ioc, "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n",
5583                              ioc->reply_free, ioc->reply_free_queue_depth,
5584                              4, sz / 1024));
5585         dinitprintk(ioc,
5586                     ioc_info(ioc, "reply_free_dma (0x%llx)\n",
5587                              (unsigned long long)ioc->reply_free_dma));
5588         total_sz += sz;
5589
5590         if (ioc->rdpq_array_enable) {
5591                 reply_post_free_array_sz = ioc->reply_queue_count *
5592                     sizeof(Mpi2IOCInitRDPQArrayEntry);
5593                 ioc->reply_post_free_array_dma_pool =
5594                     dma_pool_create("reply_post_free_array pool",
5595                     &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
5596                 if (!ioc->reply_post_free_array_dma_pool) {
5597                         dinitprintk(ioc,
5598                                     ioc_info(ioc, "reply_post_free_array pool: dma_pool_create failed\n"));
5599                         goto out;
5600                 }
5601                 ioc->reply_post_free_array =
5602                     dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
5603                     GFP_KERNEL, &ioc->reply_post_free_array_dma);
5604                 if (!ioc->reply_post_free_array) {
5605                         dinitprintk(ioc,
5606                                     ioc_info(ioc, "reply_post_free_array pool: dma_pool_alloc failed\n"));
5607                         goto out;
5608                 }
5609         }
5610         ioc->config_page_sz = 512;
5611         ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev,
5612                         ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL);
5613         if (!ioc->config_page) {
5614                 ioc_err(ioc, "config page: dma_pool_alloc failed\n");
5615                 goto out;
5616         }
5617
5618         ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n",
5619             ioc->config_page, (unsigned long long)ioc->config_page_dma,
5620             ioc->config_page_sz);
5621         total_sz += ioc->config_page_sz;
5622
5623         ioc_info(ioc, "Allocated physical memory: size(%d kB)\n",
5624                  total_sz / 1024);
5625         ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
5626                  ioc->shost->can_queue, facts->RequestCredit);
5627         ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n",
5628                  ioc->shost->sg_tablesize);
5629         return 0;
5630
5631 try_32bit_dma:
5632         _base_release_memory_pools(ioc);
5633         if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) {
5634                 /* Change dma coherent mask to 32 bit and reallocate */
5635                 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {
5636                         pr_err("Setting 32 bit coherent DMA mask Failed %s\n",
5637                             pci_name(ioc->pdev));
5638                         return -ENODEV;
5639                 }
5640         } else if (_base_reduce_hba_queue_depth(ioc) != 0)
5641                 return -ENOMEM;
5642         goto retry_allocation;
5643
5644  out:
5645         return -ENOMEM;
5646 }
5647
5648 /**
5649  * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
5650  * @ioc: Pointer to MPT_ADAPTER structure
5651  * @cooked: Request raw or cooked IOC state
5652  *
5653  * Return: all IOC Doorbell register bits if cooked==0, else just the
5654  * Doorbell bits in MPI_IOC_STATE_MASK.
5655  */
5656 u32
5657 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
5658 {
5659         u32 s, sc;
5660
5661         s = ioc->base_readl(&ioc->chip->Doorbell);
5662         sc = s & MPI2_IOC_STATE_MASK;
5663         return cooked ? sc : s;
5664 }
5665
5666 /**
5667  * _base_wait_on_iocstate - waiting on a particular ioc state
5668  * @ioc: ?
5669  * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
5670  * @timeout: timeout in second
5671  *
5672  * Return: 0 for success, non-zero for failure.
5673  */
5674 static int
5675 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
5676 {
5677         u32 count, cntdn;
5678         u32 current_state;
5679
5680         count = 0;
5681         cntdn = 1000 * timeout;
5682         do {
5683                 current_state = mpt3sas_base_get_iocstate(ioc, 1);
5684                 if (current_state == ioc_state)
5685                         return 0;
5686                 if (count && current_state == MPI2_IOC_STATE_FAULT)
5687                         break;
5688                 if (count && current_state == MPI2_IOC_STATE_COREDUMP)
5689                         break;
5690
5691                 usleep_range(1000, 1500);
5692                 count++;
5693         } while (--cntdn);
5694
5695         return current_state;
5696 }
5697
5698 /**
5699  * _base_dump_reg_set - This function will print hexdump of register set.
5700  * @ioc: per adapter object
5701  *
5702  * Returns nothing.
5703  */
5704 static inline void
5705 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc)
5706 {
5707         unsigned int i, sz = 256;
5708         u32 __iomem *reg = (u32 __iomem *)ioc->chip;
5709
5710         ioc_info(ioc, "System Register set:\n");
5711         for (i = 0; i < (sz / sizeof(u32)); i++)
5712                 pr_info("%08x: %08x\n", (i * 4), readl(&reg[i]));
5713 }
5714
5715 /**
5716  * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
5717  * a write to the doorbell)
5718  * @ioc: per adapter object
5719  * @timeout: timeout in seconds
5720  *
5721  * Return: 0 for success, non-zero for failure.
5722  *
5723  * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
5724  */
5725
5726 static int
5727 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
5728 {
5729         u32 cntdn, count;
5730         u32 int_status;
5731
5732         count = 0;
5733         cntdn = 1000 * timeout;
5734         do {
5735                 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
5736                 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5737                         dhsprintk(ioc,
5738                                   ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
5739                                            __func__, count, timeout));
5740                         return 0;
5741                 }
5742
5743                 usleep_range(1000, 1500);
5744                 count++;
5745         } while (--cntdn);
5746
5747         ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
5748                 __func__, count, int_status);
5749         return -EFAULT;
5750 }
5751
5752 static int
5753 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
5754 {
5755         u32 cntdn, count;
5756         u32 int_status;
5757
5758         count = 0;
5759         cntdn = 2000 * timeout;
5760         do {
5761                 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
5762                 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5763                         dhsprintk(ioc,
5764                                   ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
5765                                            __func__, count, timeout));
5766                         return 0;
5767                 }
5768
5769                 udelay(500);
5770                 count++;
5771         } while (--cntdn);
5772
5773         ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
5774                 __func__, count, int_status);
5775         return -EFAULT;
5776
5777 }
5778
5779 /**
5780  * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
5781  * @ioc: per adapter object
5782  * @timeout: timeout in second
5783  *
5784  * Return: 0 for success, non-zero for failure.
5785  *
5786  * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
5787  * doorbell.
5788  */
5789 static int
5790 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
5791 {
5792         u32 cntdn, count;
5793         u32 int_status;
5794         u32 doorbell;
5795
5796         count = 0;
5797         cntdn = 1000 * timeout;
5798         do {
5799                 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
5800                 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
5801                         dhsprintk(ioc,
5802                                   ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
5803                                            __func__, count, timeout));
5804                         return 0;
5805                 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5806                         doorbell = ioc->base_readl(&ioc->chip->Doorbell);
5807                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
5808                             MPI2_IOC_STATE_FAULT) {
5809                                 mpt3sas_print_fault_code(ioc, doorbell);
5810                                 return -EFAULT;
5811                         }
5812                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
5813                             MPI2_IOC_STATE_COREDUMP) {
5814                                 mpt3sas_print_coredump_info(ioc, doorbell);
5815                                 return -EFAULT;
5816                         }
5817                 } else if (int_status == 0xFFFFFFFF)
5818                         goto out;
5819
5820                 usleep_range(1000, 1500);
5821                 count++;
5822         } while (--cntdn);
5823
5824  out:
5825         ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
5826                 __func__, count, int_status);
5827         return -EFAULT;
5828 }
5829
5830 /**
5831  * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
5832  * @ioc: per adapter object
5833  * @timeout: timeout in second
5834  *
5835  * Return: 0 for success, non-zero for failure.
5836  */
5837 static int
5838 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
5839 {
5840         u32 cntdn, count;
5841         u32 doorbell_reg;
5842
5843         count = 0;
5844         cntdn = 1000 * timeout;
5845         do {
5846                 doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell);
5847                 if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
5848                         dhsprintk(ioc,
5849                                   ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
5850                                            __func__, count, timeout));
5851                         return 0;
5852                 }
5853
5854                 usleep_range(1000, 1500);
5855                 count++;
5856         } while (--cntdn);
5857
5858         ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
5859                 __func__, count, doorbell_reg);
5860         return -EFAULT;
5861 }
5862
5863 /**
5864  * _base_send_ioc_reset - send doorbell reset
5865  * @ioc: per adapter object
5866  * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
5867  * @timeout: timeout in second
5868  *
5869  * Return: 0 for success, non-zero for failure.
5870  */
5871 static int
5872 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
5873 {
5874         u32 ioc_state;
5875         int r = 0;
5876         unsigned long flags;
5877
5878         if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
5879                 ioc_err(ioc, "%s: unknown reset_type\n", __func__);
5880                 return -EFAULT;
5881         }
5882
5883         if (!(ioc->facts.IOCCapabilities &
5884            MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
5885                 return -EFAULT;
5886
5887         ioc_info(ioc, "sending message unit reset !!\n");
5888
5889         writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
5890             &ioc->chip->Doorbell);
5891         if ((_base_wait_for_doorbell_ack(ioc, 15))) {
5892                 r = -EFAULT;
5893                 goto out;
5894         }
5895
5896         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
5897         if (ioc_state) {
5898                 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
5899                         __func__, ioc_state);
5900                 r = -EFAULT;
5901                 goto out;
5902         }
5903  out:
5904         if (r != 0) {
5905                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5906                 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
5907                 /*
5908                  * Wait for IOC state CoreDump to clear only during
5909                  * HBA initialization & release time.
5910                  */
5911                 if ((ioc_state & MPI2_IOC_STATE_MASK) ==
5912                     MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 ||
5913                     ioc->fault_reset_work_q == NULL)) {
5914                         spin_unlock_irqrestore(
5915                             &ioc->ioc_reset_in_progress_lock, flags);
5916                         mpt3sas_print_coredump_info(ioc, ioc_state);
5917                         mpt3sas_base_wait_for_coredump_completion(ioc,
5918                             __func__);
5919                         spin_lock_irqsave(
5920                             &ioc->ioc_reset_in_progress_lock, flags);
5921                 }
5922                 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
5923         }
5924         ioc_info(ioc, "message unit reset: %s\n",
5925                  r == 0 ? "SUCCESS" : "FAILED");
5926         return r;
5927 }
5928
5929 /**
5930  * mpt3sas_wait_for_ioc - IOC's operational state is checked here.
5931  * @ioc: per adapter object
5932  * @timeout: timeout in seconds
5933  *
5934  * Return: Waits up to timeout seconds for the IOC to
5935  * become operational. Returns 0 if IOC is present
5936  * and operational; otherwise returns -EFAULT.
5937  */
5938
5939 int
5940 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout)
5941 {
5942         int wait_state_count = 0;
5943         u32 ioc_state;
5944
5945         do {
5946                 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5947                 if (ioc_state == MPI2_IOC_STATE_OPERATIONAL)
5948                         break;
5949                 ssleep(1);
5950                 ioc_info(ioc, "%s: waiting for operational state(count=%d)\n",
5951                                 __func__, ++wait_state_count);
5952         } while (--timeout);
5953         if (!timeout) {
5954                 ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__);
5955                 return -EFAULT;
5956         }
5957         if (wait_state_count)
5958                 ioc_info(ioc, "ioc is operational\n");
5959         return 0;
5960 }
5961
5962 /**
5963  * _base_handshake_req_reply_wait - send request thru doorbell interface
5964  * @ioc: per adapter object
5965  * @request_bytes: request length
5966  * @request: pointer having request payload
5967  * @reply_bytes: reply length
5968  * @reply: pointer to reply payload
5969  * @timeout: timeout in second
5970  *
5971  * Return: 0 for success, non-zero for failure.
5972  */
5973 static int
5974 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
5975         u32 *request, int reply_bytes, u16 *reply, int timeout)
5976 {
5977         MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
5978         int i;
5979         u8 failed;
5980         __le32 *mfp;
5981
5982         /* make sure doorbell is not in use */
5983         if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
5984                 ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__);
5985                 return -EFAULT;
5986         }
5987
5988         /* clear pending doorbell interrupts from previous state changes */
5989         if (ioc->base_readl(&ioc->chip->HostInterruptStatus) &
5990             MPI2_HIS_IOC2SYS_DB_STATUS)
5991                 writel(0, &ioc->chip->HostInterruptStatus);
5992
5993         /* send message to ioc */
5994         writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
5995             ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
5996             &ioc->chip->Doorbell);
5997
5998         if ((_base_spin_on_doorbell_int(ioc, 5))) {
5999                 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6000                         __LINE__);
6001                 return -EFAULT;
6002         }
6003         writel(0, &ioc->chip->HostInterruptStatus);
6004
6005         if ((_base_wait_for_doorbell_ack(ioc, 5))) {
6006                 ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n",
6007                         __LINE__);
6008                 return -EFAULT;
6009         }
6010
6011         /* send message 32-bits at a time */
6012         for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
6013                 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
6014                 if ((_base_wait_for_doorbell_ack(ioc, 5)))
6015                         failed = 1;
6016         }
6017
6018         if (failed) {
6019                 ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n",
6020                         __LINE__);
6021                 return -EFAULT;
6022         }
6023
6024         /* now wait for the reply */
6025         if ((_base_wait_for_doorbell_int(ioc, timeout))) {
6026                 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6027                         __LINE__);
6028                 return -EFAULT;
6029         }
6030
6031         /* read the first two 16-bits, it gives the total length of the reply */
6032         reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)
6033             & MPI2_DOORBELL_DATA_MASK);
6034         writel(0, &ioc->chip->HostInterruptStatus);
6035         if ((_base_wait_for_doorbell_int(ioc, 5))) {
6036                 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6037                         __LINE__);
6038                 return -EFAULT;
6039         }
6040         reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)
6041             & MPI2_DOORBELL_DATA_MASK);
6042         writel(0, &ioc->chip->HostInterruptStatus);
6043
6044         for (i = 2; i < default_reply->MsgLength * 2; i++)  {
6045                 if ((_base_wait_for_doorbell_int(ioc, 5))) {
6046                         ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6047                                 __LINE__);
6048                         return -EFAULT;
6049                 }
6050                 if (i >=  reply_bytes/2) /* overflow case */
6051                         ioc->base_readl(&ioc->chip->Doorbell);
6052                 else
6053                         reply[i] = le16_to_cpu(
6054                             ioc->base_readl(&ioc->chip->Doorbell)
6055                             & MPI2_DOORBELL_DATA_MASK);
6056                 writel(0, &ioc->chip->HostInterruptStatus);
6057         }
6058
6059         _base_wait_for_doorbell_int(ioc, 5);
6060         if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
6061                 dhsprintk(ioc,
6062                           ioc_info(ioc, "doorbell is in use (line=%d)\n",
6063                                    __LINE__));
6064         }
6065         writel(0, &ioc->chip->HostInterruptStatus);
6066
6067         if (ioc->logging_level & MPT_DEBUG_INIT) {
6068                 mfp = (__le32 *)reply;
6069                 pr_info("\toffset:data\n");
6070                 for (i = 0; i < reply_bytes/4; i++)
6071                         ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,
6072                             le32_to_cpu(mfp[i]));
6073         }
6074         return 0;
6075 }
6076
6077 /**
6078  * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
6079  * @ioc: per adapter object
6080  * @mpi_reply: the reply payload from FW
6081  * @mpi_request: the request payload sent to FW
6082  *
6083  * The SAS IO Unit Control Request message allows the host to perform low-level
6084  * operations, such as resets on the PHYs of the IO Unit, also allows the host
6085  * to obtain the IOC assigned device handles for a device if it has other
6086  * identifying information about the device, in addition allows the host to
6087  * remove IOC resources associated with the device.
6088  *
6089  * Return: 0 for success, non-zero for failure.
6090  */
6091 int
6092 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
6093         Mpi2SasIoUnitControlReply_t *mpi_reply,
6094         Mpi2SasIoUnitControlRequest_t *mpi_request)
6095 {
6096         u16 smid;
6097         u8 issue_reset = 0;
6098         int rc;
6099         void *request;
6100
6101         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6102
6103         mutex_lock(&ioc->base_cmds.mutex);
6104
6105         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
6106                 ioc_err(ioc, "%s: base_cmd in use\n", __func__);
6107                 rc = -EAGAIN;
6108                 goto out;
6109         }
6110
6111         rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);
6112         if (rc)
6113                 goto out;
6114
6115         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6116         if (!smid) {
6117                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6118                 rc = -EAGAIN;
6119                 goto out;
6120         }
6121
6122         rc = 0;
6123         ioc->base_cmds.status = MPT3_CMD_PENDING;
6124         request = mpt3sas_base_get_msg_frame(ioc, smid);
6125         ioc->base_cmds.smid = smid;
6126         memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
6127         if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
6128             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
6129                 ioc->ioc_link_reset_in_progress = 1;
6130         init_completion(&ioc->base_cmds.done);
6131         ioc->put_smid_default(ioc, smid);
6132         wait_for_completion_timeout(&ioc->base_cmds.done,
6133             msecs_to_jiffies(10000));
6134         if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
6135             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
6136             ioc->ioc_link_reset_in_progress)
6137                 ioc->ioc_link_reset_in_progress = 0;
6138         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6139                 mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status,
6140                     mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)/4,
6141                     issue_reset);
6142                 goto issue_host_reset;
6143         }
6144         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
6145                 memcpy(mpi_reply, ioc->base_cmds.reply,
6146                     sizeof(Mpi2SasIoUnitControlReply_t));
6147         else
6148                 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
6149         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6150         goto out;
6151
6152  issue_host_reset:
6153         if (issue_reset)
6154                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
6155         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6156         rc = -EFAULT;
6157  out:
6158         mutex_unlock(&ioc->base_cmds.mutex);
6159         return rc;
6160 }
6161
6162 /**
6163  * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
6164  * @ioc: per adapter object
6165  * @mpi_reply: the reply payload from FW
6166  * @mpi_request: the request payload sent to FW
6167  *
6168  * The SCSI Enclosure Processor request message causes the IOC to
6169  * communicate with SES devices to control LED status signals.
6170  *
6171  * Return: 0 for success, non-zero for failure.
6172  */
6173 int
6174 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
6175         Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
6176 {
6177         u16 smid;
6178         u8 issue_reset = 0;
6179         int rc;
6180         void *request;
6181
6182         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6183
6184         mutex_lock(&ioc->base_cmds.mutex);
6185
6186         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
6187                 ioc_err(ioc, "%s: base_cmd in use\n", __func__);
6188                 rc = -EAGAIN;
6189                 goto out;
6190         }
6191
6192         rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);
6193         if (rc)
6194                 goto out;
6195
6196         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6197         if (!smid) {
6198                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6199                 rc = -EAGAIN;
6200                 goto out;
6201         }
6202
6203         rc = 0;
6204         ioc->base_cmds.status = MPT3_CMD_PENDING;
6205         request = mpt3sas_base_get_msg_frame(ioc, smid);
6206         ioc->base_cmds.smid = smid;
6207         memset(request, 0, ioc->request_sz);
6208         memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
6209         init_completion(&ioc->base_cmds.done);
6210         ioc->put_smid_default(ioc, smid);
6211         wait_for_completion_timeout(&ioc->base_cmds.done,
6212             msecs_to_jiffies(10000));
6213         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6214                 mpt3sas_check_cmd_timeout(ioc,
6215                     ioc->base_cmds.status, mpi_request,
6216                     sizeof(Mpi2SepRequest_t)/4, issue_reset);
6217                 goto issue_host_reset;
6218         }
6219         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
6220                 memcpy(mpi_reply, ioc->base_cmds.reply,
6221                     sizeof(Mpi2SepReply_t));
6222         else
6223                 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
6224         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6225         goto out;
6226
6227  issue_host_reset:
6228         if (issue_reset)
6229                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
6230         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6231         rc = -EFAULT;
6232  out:
6233         mutex_unlock(&ioc->base_cmds.mutex);
6234         return rc;
6235 }
6236
6237 /**
6238  * _base_get_port_facts - obtain port facts reply and save in ioc
6239  * @ioc: per adapter object
6240  * @port: ?
6241  *
6242  * Return: 0 for success, non-zero for failure.
6243  */
6244 static int
6245 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
6246 {
6247         Mpi2PortFactsRequest_t mpi_request;
6248         Mpi2PortFactsReply_t mpi_reply;
6249         struct mpt3sas_port_facts *pfacts;
6250         int mpi_reply_sz, mpi_request_sz, r;
6251
6252         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6253
6254         mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
6255         mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
6256         memset(&mpi_request, 0, mpi_request_sz);
6257         mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
6258         mpi_request.PortNumber = port;
6259         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
6260             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
6261
6262         if (r != 0) {
6263                 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
6264                 return r;
6265         }
6266
6267         pfacts = &ioc->pfacts[port];
6268         memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
6269         pfacts->PortNumber = mpi_reply.PortNumber;
6270         pfacts->VP_ID = mpi_reply.VP_ID;
6271         pfacts->VF_ID = mpi_reply.VF_ID;
6272         pfacts->MaxPostedCmdBuffers =
6273             le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
6274
6275         return 0;
6276 }
6277
6278 /**
6279  * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
6280  * @ioc: per adapter object
6281  * @timeout:
6282  *
6283  * Return: 0 for success, non-zero for failure.
6284  */
6285 static int
6286 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
6287 {
6288         u32 ioc_state;
6289         int rc;
6290
6291         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6292
6293         if (ioc->pci_error_recovery) {
6294                 dfailprintk(ioc,
6295                             ioc_info(ioc, "%s: host in pci error recovery\n",
6296                                      __func__));
6297                 return -EFAULT;
6298         }
6299
6300         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6301         dhsprintk(ioc,
6302                   ioc_info(ioc, "%s: ioc_state(0x%08x)\n",
6303                            __func__, ioc_state));
6304
6305         if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
6306             (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
6307                 return 0;
6308
6309         if (ioc_state & MPI2_DOORBELL_USED) {
6310                 dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));
6311                 goto issue_diag_reset;
6312         }
6313
6314         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
6315                 mpt3sas_print_fault_code(ioc, ioc_state &
6316                     MPI2_DOORBELL_DATA_MASK);
6317                 goto issue_diag_reset;
6318         } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
6319             MPI2_IOC_STATE_COREDUMP) {
6320                 ioc_info(ioc,
6321                     "%s: Skipping the diag reset here. (ioc_state=0x%x)\n",
6322                     __func__, ioc_state);
6323                 return -EFAULT;
6324         }
6325
6326         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
6327         if (ioc_state) {
6328                 dfailprintk(ioc,
6329                             ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
6330                                      __func__, ioc_state));
6331                 return -EFAULT;
6332         }
6333
6334  issue_diag_reset:
6335         rc = _base_diag_reset(ioc);
6336         return rc;
6337 }
6338
6339 /**
6340  * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
6341  * @ioc: per adapter object
6342  *
6343  * Return: 0 for success, non-zero for failure.
6344  */
6345 static int
6346 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
6347 {
6348         Mpi2IOCFactsRequest_t mpi_request;
6349         Mpi2IOCFactsReply_t mpi_reply;
6350         struct mpt3sas_facts *facts;
6351         int mpi_reply_sz, mpi_request_sz, r;
6352
6353         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6354
6355         r = _base_wait_for_iocstate(ioc, 10);
6356         if (r) {
6357                 dfailprintk(ioc,
6358                             ioc_info(ioc, "%s: failed getting to correct state\n",
6359                                      __func__));
6360                 return r;
6361         }
6362         mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
6363         mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
6364         memset(&mpi_request, 0, mpi_request_sz);
6365         mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
6366         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
6367             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
6368
6369         if (r != 0) {
6370                 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
6371                 return r;
6372         }
6373
6374         facts = &ioc->facts;
6375         memset(facts, 0, sizeof(struct mpt3sas_facts));
6376         facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
6377         facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
6378         facts->VP_ID = mpi_reply.VP_ID;
6379         facts->VF_ID = mpi_reply.VF_ID;
6380         facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
6381         facts->MaxChainDepth = mpi_reply.MaxChainDepth;
6382         facts->WhoInit = mpi_reply.WhoInit;
6383         facts->NumberOfPorts = mpi_reply.NumberOfPorts;
6384         facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
6385         if (ioc->msix_enable && (facts->MaxMSIxVectors <=
6386             MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc)))
6387                 ioc->combined_reply_queue = 0;
6388         facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
6389         facts->MaxReplyDescriptorPostQueueDepth =
6390             le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
6391         facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
6392         facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
6393         if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
6394                 ioc->ir_firmware = 1;
6395         if ((facts->IOCCapabilities &
6396               MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices))
6397                 ioc->rdpq_array_capable = 1;
6398         if ((facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ)
6399             && ioc->is_aero_ioc)
6400                 ioc->atomic_desc_capable = 1;
6401         facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
6402         facts->IOCRequestFrameSize =
6403             le16_to_cpu(mpi_reply.IOCRequestFrameSize);
6404         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
6405                 facts->IOCMaxChainSegmentSize =
6406                         le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize);
6407         }
6408         facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
6409         facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
6410         ioc->shost->max_id = -1;
6411         facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
6412         facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
6413         facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
6414         facts->HighPriorityCredit =
6415             le16_to_cpu(mpi_reply.HighPriorityCredit);
6416         facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
6417         facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
6418         facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize;
6419
6420         /*
6421          * Get the Page Size from IOC Facts. If it's 0, default to 4k.
6422          */
6423         ioc->page_size = 1 << facts->CurrentHostPageSize;
6424         if (ioc->page_size == 1) {
6425                 ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n");
6426                 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;
6427         }
6428         dinitprintk(ioc,
6429                     ioc_info(ioc, "CurrentHostPageSize(%d)\n",
6430                              facts->CurrentHostPageSize));
6431
6432         dinitprintk(ioc,
6433                     ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n",
6434                              facts->RequestCredit, facts->MaxChainDepth));
6435         dinitprintk(ioc,
6436                     ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n",
6437                              facts->IOCRequestFrameSize * 4,
6438                              facts->ReplyFrameSize * 4));
6439         return 0;
6440 }
6441
6442 /**
6443  * _base_send_ioc_init - send ioc_init to firmware
6444  * @ioc: per adapter object
6445  *
6446  * Return: 0 for success, non-zero for failure.
6447  */
6448 static int
6449 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
6450 {
6451         Mpi2IOCInitRequest_t mpi_request;
6452         Mpi2IOCInitReply_t mpi_reply;
6453         int i, r = 0;
6454         ktime_t current_time;
6455         u16 ioc_status;
6456         u32 reply_post_free_array_sz = 0;
6457
6458         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6459
6460         memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
6461         mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
6462         mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
6463         mpi_request.VF_ID = 0; /* TODO */
6464         mpi_request.VP_ID = 0;
6465         mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
6466         mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
6467         mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K;
6468
6469         if (_base_is_controller_msix_enabled(ioc))
6470                 mpi_request.HostMSIxVectors = ioc->reply_queue_count;
6471         mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
6472         mpi_request.ReplyDescriptorPostQueueDepth =
6473             cpu_to_le16(ioc->reply_post_queue_depth);
6474         mpi_request.ReplyFreeQueueDepth =
6475             cpu_to_le16(ioc->reply_free_queue_depth);
6476
6477         mpi_request.SenseBufferAddressHigh =
6478             cpu_to_le32((u64)ioc->sense_dma >> 32);
6479         mpi_request.SystemReplyAddressHigh =
6480             cpu_to_le32((u64)ioc->reply_dma >> 32);
6481         mpi_request.SystemRequestFrameBaseAddress =
6482             cpu_to_le64((u64)ioc->request_dma);
6483         mpi_request.ReplyFreeQueueAddress =
6484             cpu_to_le64((u64)ioc->reply_free_dma);
6485
6486         if (ioc->rdpq_array_enable) {
6487                 reply_post_free_array_sz = ioc->reply_queue_count *
6488                     sizeof(Mpi2IOCInitRDPQArrayEntry);
6489                 memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
6490                 for (i = 0; i < ioc->reply_queue_count; i++)
6491                         ioc->reply_post_free_array[i].RDPQBaseAddress =
6492                             cpu_to_le64(
6493                                 (u64)ioc->reply_post[i].reply_post_free_dma);
6494                 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
6495                 mpi_request.ReplyDescriptorPostQueueAddress =
6496                     cpu_to_le64((u64)ioc->reply_post_free_array_dma);
6497         } else {
6498                 mpi_request.ReplyDescriptorPostQueueAddress =
6499                     cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
6500         }
6501
6502         /*
6503          * Set the flag to enable CoreDump state feature in IOC firmware.
6504          */
6505         mpi_request.ConfigurationFlags |=
6506             cpu_to_le16(MPI26_IOCINIT_CFGFLAGS_COREDUMP_ENABLE);
6507
6508         /* This time stamp specifies number of milliseconds
6509          * since epoch ~ midnight January 1, 1970.
6510          */
6511         current_time = ktime_get_real();
6512         mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time));
6513
6514         if (ioc->logging_level & MPT_DEBUG_INIT) {
6515                 __le32 *mfp;
6516                 int i;
6517
6518                 mfp = (__le32 *)&mpi_request;
6519                 ioc_info(ioc, "\toffset:data\n");
6520                 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
6521                         ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,
6522                             le32_to_cpu(mfp[i]));
6523         }
6524
6525         r = _base_handshake_req_reply_wait(ioc,
6526             sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
6527             sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30);
6528
6529         if (r != 0) {
6530                 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
6531                 return r;
6532         }
6533
6534         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6535         if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
6536             mpi_reply.IOCLogInfo) {
6537                 ioc_err(ioc, "%s: failed\n", __func__);
6538                 r = -EIO;
6539         }
6540
6541         return r;
6542 }
6543
6544 /**
6545  * mpt3sas_port_enable_done - command completion routine for port enable
6546  * @ioc: per adapter object
6547  * @smid: system request message index
6548  * @msix_index: MSIX table index supplied by the OS
6549  * @reply: reply message frame(lower 32bit addr)
6550  *
6551  * Return: 1 meaning mf should be freed from _base_interrupt
6552  *          0 means the mf is freed from this function.
6553  */
6554 u8
6555 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
6556         u32 reply)
6557 {
6558         MPI2DefaultReply_t *mpi_reply;
6559         u16 ioc_status;
6560
6561         if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
6562                 return 1;
6563
6564         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
6565         if (!mpi_reply)
6566                 return 1;
6567
6568         if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
6569                 return 1;
6570
6571         ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
6572         ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
6573         ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
6574         memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
6575         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
6576         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6577                 ioc->port_enable_failed = 1;
6578
6579         if (ioc->is_driver_loading) {
6580                 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
6581                         mpt3sas_port_enable_complete(ioc);
6582                         return 1;
6583                 } else {
6584                         ioc->start_scan_failed = ioc_status;
6585                         ioc->start_scan = 0;
6586                         return 1;
6587                 }
6588         }
6589         complete(&ioc->port_enable_cmds.done);
6590         return 1;
6591 }
6592
6593 /**
6594  * _base_send_port_enable - send port_enable(discovery stuff) to firmware
6595  * @ioc: per adapter object
6596  *
6597  * Return: 0 for success, non-zero for failure.
6598  */
6599 static int
6600 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
6601 {
6602         Mpi2PortEnableRequest_t *mpi_request;
6603         Mpi2PortEnableReply_t *mpi_reply;
6604         int r = 0;
6605         u16 smid;
6606         u16 ioc_status;
6607
6608         ioc_info(ioc, "sending port enable !!\n");
6609
6610         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
6611                 ioc_err(ioc, "%s: internal command already in use\n", __func__);
6612                 return -EAGAIN;
6613         }
6614
6615         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
6616         if (!smid) {
6617                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6618                 return -EAGAIN;
6619         }
6620
6621         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
6622         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
6623         ioc->port_enable_cmds.smid = smid;
6624         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
6625         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
6626
6627         init_completion(&ioc->port_enable_cmds.done);
6628         ioc->put_smid_default(ioc, smid);
6629         wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
6630         if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
6631                 ioc_err(ioc, "%s: timeout\n", __func__);
6632                 _debug_dump_mf(mpi_request,
6633                     sizeof(Mpi2PortEnableRequest_t)/4);
6634                 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
6635                         r = -EFAULT;
6636                 else
6637                         r = -ETIME;
6638                 goto out;
6639         }
6640
6641         mpi_reply = ioc->port_enable_cmds.reply;
6642         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
6643         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6644                 ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n",
6645                         __func__, ioc_status);
6646                 r = -EFAULT;
6647                 goto out;
6648         }
6649
6650  out:
6651         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
6652         ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED");
6653         return r;
6654 }
6655
6656 /**
6657  * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
6658  * @ioc: per adapter object
6659  *
6660  * Return: 0 for success, non-zero for failure.
6661  */
6662 int
6663 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
6664 {
6665         Mpi2PortEnableRequest_t *mpi_request;
6666         u16 smid;
6667
6668         ioc_info(ioc, "sending port enable !!\n");
6669
6670         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
6671                 ioc_err(ioc, "%s: internal command already in use\n", __func__);
6672                 return -EAGAIN;
6673         }
6674
6675         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
6676         if (!smid) {
6677                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6678                 return -EAGAIN;
6679         }
6680
6681         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
6682         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
6683         ioc->port_enable_cmds.smid = smid;
6684         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
6685         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
6686
6687         ioc->put_smid_default(ioc, smid);
6688         return 0;
6689 }
6690
6691 /**
6692  * _base_determine_wait_on_discovery - desposition
6693  * @ioc: per adapter object
6694  *
6695  * Decide whether to wait on discovery to complete. Used to either
6696  * locate boot device, or report volumes ahead of physical devices.
6697  *
6698  * Return: 1 for wait, 0 for don't wait.
6699  */
6700 static int
6701 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
6702 {
6703         /* We wait for discovery to complete if IR firmware is loaded.
6704          * The sas topology events arrive before PD events, so we need time to
6705          * turn on the bit in ioc->pd_handles to indicate PD
6706          * Also, it maybe required to report Volumes ahead of physical
6707          * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
6708          */
6709         if (ioc->ir_firmware)
6710                 return 1;
6711
6712         /* if no Bios, then we don't need to wait */
6713         if (!ioc->bios_pg3.BiosVersion)
6714                 return 0;
6715
6716         /* Bios is present, then we drop down here.
6717          *
6718          * If there any entries in the Bios Page 2, then we wait
6719          * for discovery to complete.
6720          */
6721
6722         /* Current Boot Device */
6723         if ((ioc->bios_pg2.CurrentBootDeviceForm &
6724             MPI2_BIOSPAGE2_FORM_MASK) ==
6725             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
6726         /* Request Boot Device */
6727            (ioc->bios_pg2.ReqBootDeviceForm &
6728             MPI2_BIOSPAGE2_FORM_MASK) ==
6729             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
6730         /* Alternate Request Boot Device */
6731            (ioc->bios_pg2.ReqAltBootDeviceForm &
6732             MPI2_BIOSPAGE2_FORM_MASK) ==
6733             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
6734                 return 0;
6735
6736         return 1;
6737 }
6738
6739 /**
6740  * _base_unmask_events - turn on notification for this event
6741  * @ioc: per adapter object
6742  * @event: firmware event
6743  *
6744  * The mask is stored in ioc->event_masks.
6745  */
6746 static void
6747 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
6748 {
6749         u32 desired_event;
6750
6751         if (event >= 128)
6752                 return;
6753
6754         desired_event = (1 << (event % 32));
6755
6756         if (event < 32)
6757                 ioc->event_masks[0] &= ~desired_event;
6758         else if (event < 64)
6759                 ioc->event_masks[1] &= ~desired_event;
6760         else if (event < 96)
6761                 ioc->event_masks[2] &= ~desired_event;
6762         else if (event < 128)
6763                 ioc->event_masks[3] &= ~desired_event;
6764 }
6765
6766 /**
6767  * _base_event_notification - send event notification
6768  * @ioc: per adapter object
6769  *
6770  * Return: 0 for success, non-zero for failure.
6771  */
6772 static int
6773 _base_event_notification(struct MPT3SAS_ADAPTER *ioc)
6774 {
6775         Mpi2EventNotificationRequest_t *mpi_request;
6776         u16 smid;
6777         int r = 0;
6778         int i;
6779
6780         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6781
6782         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
6783                 ioc_err(ioc, "%s: internal command already in use\n", __func__);
6784                 return -EAGAIN;
6785         }
6786
6787         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6788         if (!smid) {
6789                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6790                 return -EAGAIN;
6791         }
6792         ioc->base_cmds.status = MPT3_CMD_PENDING;
6793         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
6794         ioc->base_cmds.smid = smid;
6795         memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
6796         mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
6797         mpi_request->VF_ID = 0; /* TODO */
6798         mpi_request->VP_ID = 0;
6799         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
6800                 mpi_request->EventMasks[i] =
6801                     cpu_to_le32(ioc->event_masks[i]);
6802         init_completion(&ioc->base_cmds.done);
6803         ioc->put_smid_default(ioc, smid);
6804         wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
6805         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6806                 ioc_err(ioc, "%s: timeout\n", __func__);
6807                 _debug_dump_mf(mpi_request,
6808                     sizeof(Mpi2EventNotificationRequest_t)/4);
6809                 if (ioc->base_cmds.status & MPT3_CMD_RESET)
6810                         r = -EFAULT;
6811                 else
6812                         r = -ETIME;
6813         } else
6814                 dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__));
6815         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6816         return r;
6817 }
6818
6819 /**
6820  * mpt3sas_base_validate_event_type - validating event types
6821  * @ioc: per adapter object
6822  * @event_type: firmware event
6823  *
6824  * This will turn on firmware event notification when application
6825  * ask for that event. We don't mask events that are already enabled.
6826  */
6827 void
6828 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
6829 {
6830         int i, j;
6831         u32 event_mask, desired_event;
6832         u8 send_update_to_fw;
6833
6834         for (i = 0, send_update_to_fw = 0; i <
6835             MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
6836                 event_mask = ~event_type[i];
6837                 desired_event = 1;
6838                 for (j = 0; j < 32; j++) {
6839                         if (!(event_mask & desired_event) &&
6840                             (ioc->event_masks[i] & desired_event)) {
6841                                 ioc->event_masks[i] &= ~desired_event;
6842                                 send_update_to_fw = 1;
6843                         }
6844                         desired_event = (desired_event << 1);
6845                 }
6846         }
6847
6848         if (!send_update_to_fw)
6849                 return;
6850
6851         mutex_lock(&ioc->base_cmds.mutex);
6852         _base_event_notification(ioc);
6853         mutex_unlock(&ioc->base_cmds.mutex);
6854 }
6855
6856 /**
6857  * _base_diag_reset - the "big hammer" start of day reset
6858  * @ioc: per adapter object
6859  *
6860  * Return: 0 for success, non-zero for failure.
6861  */
6862 static int
6863 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
6864 {
6865         u32 host_diagnostic;
6866         u32 ioc_state;
6867         u32 count;
6868         u32 hcb_size;
6869
6870         ioc_info(ioc, "sending diag reset !!\n");
6871
6872         pci_cfg_access_lock(ioc->pdev);
6873
6874         drsprintk(ioc, ioc_info(ioc, "clear interrupts\n"));
6875
6876         count = 0;
6877         do {
6878                 /* Write magic sequence to WriteSequence register
6879                  * Loop until in diagnostic mode
6880                  */
6881                 drsprintk(ioc, ioc_info(ioc, "write magic sequence\n"));
6882                 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
6883                 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
6884                 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
6885                 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
6886                 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
6887                 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
6888                 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
6889
6890                 /* wait 100 msec */
6891                 msleep(100);
6892
6893                 if (count++ > 20) {
6894                         ioc_info(ioc,
6895                             "Stop writing magic sequence after 20 retries\n");
6896                         _base_dump_reg_set(ioc);
6897                         goto out;
6898                 }
6899
6900                 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);
6901                 drsprintk(ioc,
6902                           ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
6903                                    count, host_diagnostic));
6904
6905         } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
6906
6907         hcb_size = ioc->base_readl(&ioc->chip->HCBSize);
6908
6909         drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n"));
6910         writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
6911              &ioc->chip->HostDiagnostic);
6912
6913         /*This delay allows the chip PCIe hardware time to finish reset tasks*/
6914         msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
6915
6916         /* Approximately 300 second max wait */
6917         for (count = 0; count < (300000000 /
6918                 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
6919
6920                 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);
6921
6922                 if (host_diagnostic == 0xFFFFFFFF) {
6923                         ioc_info(ioc,
6924                             "Invalid host diagnostic register value\n");
6925                         _base_dump_reg_set(ioc);
6926                         goto out;
6927                 }
6928                 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
6929                         break;
6930
6931                 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
6932         }
6933
6934         if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
6935
6936                 drsprintk(ioc,
6937                           ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n"));
6938                 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
6939                 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
6940                 writel(host_diagnostic, &ioc->chip->HostDiagnostic);
6941
6942                 drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n"));
6943                 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
6944                     &ioc->chip->HCBSize);
6945         }
6946
6947         drsprintk(ioc, ioc_info(ioc, "restart the adapter\n"));
6948         writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
6949             &ioc->chip->HostDiagnostic);
6950
6951         drsprintk(ioc,
6952                   ioc_info(ioc, "disable writes to the diagnostic register\n"));
6953         writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
6954
6955         drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n"));
6956         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
6957         if (ioc_state) {
6958                 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
6959                         __func__, ioc_state);
6960                 _base_dump_reg_set(ioc);
6961                 goto out;
6962         }
6963
6964         pci_cfg_access_unlock(ioc->pdev);
6965         ioc_info(ioc, "diag reset: SUCCESS\n");
6966         return 0;
6967
6968  out:
6969         pci_cfg_access_unlock(ioc->pdev);
6970         ioc_err(ioc, "diag reset: FAILED\n");
6971         return -EFAULT;
6972 }
6973
6974 /**
6975  * _base_make_ioc_ready - put controller in READY state
6976  * @ioc: per adapter object
6977  * @type: FORCE_BIG_HAMMER or SOFT_RESET
6978  *
6979  * Return: 0 for success, non-zero for failure.
6980  */
6981 static int
6982 _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
6983 {
6984         u32 ioc_state;
6985         int rc;
6986         int count;
6987
6988         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6989
6990         if (ioc->pci_error_recovery)
6991                 return 0;
6992
6993         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6994         dhsprintk(ioc,
6995                   ioc_info(ioc, "%s: ioc_state(0x%08x)\n",
6996                            __func__, ioc_state));
6997
6998         /* if in RESET state, it should move to READY state shortly */
6999         count = 0;
7000         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
7001                 while ((ioc_state & MPI2_IOC_STATE_MASK) !=
7002                     MPI2_IOC_STATE_READY) {
7003                         if (count++ == 10) {
7004                                 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
7005                                         __func__, ioc_state);
7006                                 return -EFAULT;
7007                         }
7008                         ssleep(1);
7009                         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
7010                 }
7011         }
7012
7013         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
7014                 return 0;
7015
7016         if (ioc_state & MPI2_DOORBELL_USED) {
7017                 ioc_info(ioc, "unexpected doorbell active!\n");
7018                 goto issue_diag_reset;
7019         }
7020
7021         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
7022                 mpt3sas_print_fault_code(ioc, ioc_state &
7023                     MPI2_DOORBELL_DATA_MASK);
7024                 goto issue_diag_reset;
7025         }
7026
7027         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) {
7028                 /*
7029                  * if host reset is invoked while watch dog thread is waiting
7030                  * for IOC state to be changed to Fault state then driver has
7031                  * to wait here for CoreDump state to clear otherwise reset
7032                  * will be issued to the FW and FW move the IOC state to
7033                  * reset state without copying the FW logs to coredump region.
7034                  */
7035                 if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) {
7036                         mpt3sas_print_coredump_info(ioc, ioc_state &
7037                             MPI2_DOORBELL_DATA_MASK);
7038                         mpt3sas_base_wait_for_coredump_completion(ioc,
7039                             __func__);
7040                 }
7041                 goto issue_diag_reset;
7042         }
7043
7044         if (type == FORCE_BIG_HAMMER)
7045                 goto issue_diag_reset;
7046
7047         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
7048                 if (!(_base_send_ioc_reset(ioc,
7049                     MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
7050                         return 0;
7051         }
7052
7053  issue_diag_reset:
7054         rc = _base_diag_reset(ioc);
7055         return rc;
7056 }
7057
7058 /**
7059  * _base_make_ioc_operational - put controller in OPERATIONAL state
7060  * @ioc: per adapter object
7061  *
7062  * Return: 0 for success, non-zero for failure.
7063  */
7064 static int
7065 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
7066 {
7067         int r, i, index, rc;
7068         unsigned long   flags;
7069         u32 reply_address;
7070         u16 smid;
7071         struct _tr_list *delayed_tr, *delayed_tr_next;
7072         struct _sc_list *delayed_sc, *delayed_sc_next;
7073         struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next;
7074         u8 hide_flag;
7075         struct adapter_reply_queue *reply_q;
7076         Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig;
7077
7078         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7079
7080         /* clean the delayed target reset list */
7081         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
7082             &ioc->delayed_tr_list, list) {
7083                 list_del(&delayed_tr->list);
7084                 kfree(delayed_tr);
7085         }
7086
7087
7088         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
7089             &ioc->delayed_tr_volume_list, list) {
7090                 list_del(&delayed_tr->list);
7091                 kfree(delayed_tr);
7092         }
7093
7094         list_for_each_entry_safe(delayed_sc, delayed_sc_next,
7095             &ioc->delayed_sc_list, list) {
7096                 list_del(&delayed_sc->list);
7097                 kfree(delayed_sc);
7098         }
7099
7100         list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next,
7101             &ioc->delayed_event_ack_list, list) {
7102                 list_del(&delayed_event_ack->list);
7103                 kfree(delayed_event_ack);
7104         }
7105
7106         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
7107
7108         /* hi-priority queue */
7109         INIT_LIST_HEAD(&ioc->hpr_free_list);
7110         smid = ioc->hi_priority_smid;
7111         for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
7112                 ioc->hpr_lookup[i].cb_idx = 0xFF;
7113                 ioc->hpr_lookup[i].smid = smid;
7114                 list_add_tail(&ioc->hpr_lookup[i].tracker_list,
7115                     &ioc->hpr_free_list);
7116         }
7117
7118         /* internal queue */
7119         INIT_LIST_HEAD(&ioc->internal_free_list);
7120         smid = ioc->internal_smid;
7121         for (i = 0; i < ioc->internal_depth; i++, smid++) {
7122                 ioc->internal_lookup[i].cb_idx = 0xFF;
7123                 ioc->internal_lookup[i].smid = smid;
7124                 list_add_tail(&ioc->internal_lookup[i].tracker_list,
7125                     &ioc->internal_free_list);
7126         }
7127
7128         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
7129
7130         /* initialize Reply Free Queue */
7131         for (i = 0, reply_address = (u32)ioc->reply_dma ;
7132             i < ioc->reply_free_queue_depth ; i++, reply_address +=
7133             ioc->reply_sz) {
7134                 ioc->reply_free[i] = cpu_to_le32(reply_address);
7135                 if (ioc->is_mcpu_endpoint)
7136                         _base_clone_reply_to_sys_mem(ioc,
7137                                         reply_address, i);
7138         }
7139
7140         /* initialize reply queues */
7141         if (ioc->is_driver_loading)
7142                 _base_assign_reply_queues(ioc);
7143
7144         /* initialize Reply Post Free Queue */
7145         index = 0;
7146         reply_post_free_contig = ioc->reply_post[0].reply_post_free;
7147         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
7148                 /*
7149                  * If RDPQ is enabled, switch to the next allocation.
7150                  * Otherwise advance within the contiguous region.
7151                  */
7152                 if (ioc->rdpq_array_enable) {
7153                         reply_q->reply_post_free =
7154                                 ioc->reply_post[index++].reply_post_free;
7155                 } else {
7156                         reply_q->reply_post_free = reply_post_free_contig;
7157                         reply_post_free_contig += ioc->reply_post_queue_depth;
7158                 }
7159
7160                 reply_q->reply_post_host_index = 0;
7161                 for (i = 0; i < ioc->reply_post_queue_depth; i++)
7162                         reply_q->reply_post_free[i].Words =
7163                             cpu_to_le64(ULLONG_MAX);
7164                 if (!_base_is_controller_msix_enabled(ioc))
7165                         goto skip_init_reply_post_free_queue;
7166         }
7167  skip_init_reply_post_free_queue:
7168
7169         r = _base_send_ioc_init(ioc);
7170         if (r) {
7171                 /*
7172                  * No need to check IOC state for fault state & issue
7173                  * diag reset during host reset. This check is need
7174                  * only during driver load time.
7175                  */
7176                 if (!ioc->is_driver_loading)
7177                         return r;
7178
7179                 rc = _base_check_for_fault_and_issue_reset(ioc);
7180                 if (rc || (_base_send_ioc_init(ioc)))
7181                         return r;
7182         }
7183
7184         /* initialize reply free host index */
7185         ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
7186         writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
7187
7188         /* initialize reply post host index */
7189         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
7190                 if (ioc->combined_reply_queue)
7191                         writel((reply_q->msix_index & 7)<<
7192                            MPI2_RPHI_MSIX_INDEX_SHIFT,
7193                            ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
7194                 else
7195                         writel(reply_q->msix_index <<
7196                                 MPI2_RPHI_MSIX_INDEX_SHIFT,
7197                                 &ioc->chip->ReplyPostHostIndex);
7198
7199                 if (!_base_is_controller_msix_enabled(ioc))
7200                         goto skip_init_reply_post_host_index;
7201         }
7202
7203  skip_init_reply_post_host_index:
7204
7205         mpt3sas_base_unmask_interrupts(ioc);
7206
7207         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
7208                 r = _base_display_fwpkg_version(ioc);
7209                 if (r)
7210                         return r;
7211         }
7212
7213         _base_static_config_pages(ioc);
7214         r = _base_event_notification(ioc);
7215         if (r)
7216                 return r;
7217
7218         if (ioc->is_driver_loading) {
7219
7220                 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
7221                     == 0x80) {
7222                         hide_flag = (u8) (
7223                             le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &
7224                             MFG_PAGE10_HIDE_SSDS_MASK);
7225                         if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK)
7226                                 ioc->mfg_pg10_hide_flag = hide_flag;
7227                 }
7228
7229                 ioc->wait_for_discovery_to_complete =
7230                     _base_determine_wait_on_discovery(ioc);
7231
7232                 return r; /* scan_start and scan_finished support */
7233         }
7234
7235         r = _base_send_port_enable(ioc);
7236         if (r)
7237                 return r;
7238
7239         return r;
7240 }
7241
7242 /**
7243  * mpt3sas_base_free_resources - free resources controller resources
7244  * @ioc: per adapter object
7245  */
7246 void
7247 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
7248 {
7249         dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7250
7251         /* synchronizing freeing resource with pci_access_mutex lock */
7252         mutex_lock(&ioc->pci_access_mutex);
7253         if (ioc->chip_phys && ioc->chip) {
7254                 mpt3sas_base_mask_interrupts(ioc);
7255                 ioc->shost_recovery = 1;
7256                 _base_make_ioc_ready(ioc, SOFT_RESET);
7257                 ioc->shost_recovery = 0;
7258         }
7259
7260         mpt3sas_base_unmap_resources(ioc);
7261         mutex_unlock(&ioc->pci_access_mutex);
7262         return;
7263 }
7264
7265 /**
7266  * mpt3sas_base_attach - attach controller instance
7267  * @ioc: per adapter object
7268  *
7269  * Return: 0 for success, non-zero for failure.
7270  */
7271 int
7272 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
7273 {
7274         int r, i, rc;
7275         int cpu_id, last_cpu_id = 0;
7276
7277         dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7278
7279         /* setup cpu_msix_table */
7280         ioc->cpu_count = num_online_cpus();
7281         for_each_online_cpu(cpu_id)
7282                 last_cpu_id = cpu_id;
7283         ioc->cpu_msix_table_sz = last_cpu_id + 1;
7284         ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
7285         ioc->reply_queue_count = 1;
7286         if (!ioc->cpu_msix_table) {
7287                 ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n");
7288                 r = -ENOMEM;
7289                 goto out_free_resources;
7290         }
7291
7292         if (ioc->is_warpdrive) {
7293                 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,
7294                     sizeof(resource_size_t *), GFP_KERNEL);
7295                 if (!ioc->reply_post_host_index) {
7296                         ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n");
7297                         r = -ENOMEM;
7298                         goto out_free_resources;
7299                 }
7300         }
7301
7302         ioc->smp_affinity_enable = smp_affinity_enable;
7303
7304         ioc->rdpq_array_enable_assigned = 0;
7305         ioc->use_32bit_dma = false;
7306         ioc->dma_mask = 64;
7307         if (ioc->is_aero_ioc)
7308                 ioc->base_readl = &_base_readl_aero;
7309         else
7310                 ioc->base_readl = &_base_readl;
7311         r = mpt3sas_base_map_resources(ioc);
7312         if (r)
7313                 goto out_free_resources;
7314
7315         pci_set_drvdata(ioc->pdev, ioc->shost);
7316         r = _base_get_ioc_facts(ioc);
7317         if (r) {
7318                 rc = _base_check_for_fault_and_issue_reset(ioc);
7319                 if (rc || (_base_get_ioc_facts(ioc)))
7320                         goto out_free_resources;
7321         }
7322
7323         switch (ioc->hba_mpi_version_belonged) {
7324         case MPI2_VERSION:
7325                 ioc->build_sg_scmd = &_base_build_sg_scmd;
7326                 ioc->build_sg = &_base_build_sg;
7327                 ioc->build_zero_len_sge = &_base_build_zero_len_sge;
7328                 ioc->get_msix_index_for_smlio = &_base_get_msix_index;
7329                 break;
7330         case MPI25_VERSION:
7331         case MPI26_VERSION:
7332                 /*
7333                  * In SAS3.0,
7334                  * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
7335                  * Target Status - all require the IEEE formated scatter gather
7336                  * elements.
7337                  */
7338                 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
7339                 ioc->build_sg = &_base_build_sg_ieee;
7340                 ioc->build_nvme_prp = &_base_build_nvme_prp;
7341                 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
7342                 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
7343                 if (ioc->high_iops_queues)
7344                         ioc->get_msix_index_for_smlio =
7345                                         &_base_get_high_iops_msix_index;
7346                 else
7347                         ioc->get_msix_index_for_smlio = &_base_get_msix_index;
7348                 break;
7349         }
7350         if (ioc->atomic_desc_capable) {
7351                 ioc->put_smid_default = &_base_put_smid_default_atomic;
7352                 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;
7353                 ioc->put_smid_fast_path =
7354                                 &_base_put_smid_fast_path_atomic;
7355                 ioc->put_smid_hi_priority =
7356                                 &_base_put_smid_hi_priority_atomic;
7357         } else {
7358                 ioc->put_smid_default = &_base_put_smid_default;
7359                 ioc->put_smid_fast_path = &_base_put_smid_fast_path;
7360                 ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;
7361                 if (ioc->is_mcpu_endpoint)
7362                         ioc->put_smid_scsi_io =
7363                                 &_base_put_smid_mpi_ep_scsi_io;
7364                 else
7365                         ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
7366         }
7367         /*
7368          * These function pointers for other requests that don't
7369          * the require IEEE scatter gather elements.
7370          *
7371          * For example Configuration Pages and SAS IOUNIT Control don't.
7372          */
7373         ioc->build_sg_mpi = &_base_build_sg;
7374         ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
7375
7376         r = _base_make_ioc_ready(ioc, SOFT_RESET);
7377         if (r)
7378                 goto out_free_resources;
7379
7380         ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
7381             sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
7382         if (!ioc->pfacts) {
7383                 r = -ENOMEM;
7384                 goto out_free_resources;
7385         }
7386
7387         for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
7388                 r = _base_get_port_facts(ioc, i);
7389                 if (r) {
7390                         rc = _base_check_for_fault_and_issue_reset(ioc);
7391                         if (rc || (_base_get_port_facts(ioc, i)))
7392                                 goto out_free_resources;
7393                 }
7394         }
7395
7396         r = _base_allocate_memory_pools(ioc);
7397         if (r)
7398                 goto out_free_resources;
7399
7400         if (irqpoll_weight > 0)
7401                 ioc->thresh_hold = irqpoll_weight;
7402         else
7403                 ioc->thresh_hold = ioc->hba_queue_depth/4;
7404
7405         _base_init_irqpolls(ioc);
7406         init_waitqueue_head(&ioc->reset_wq);
7407
7408         /* allocate memory pd handle bitmask list */
7409         ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
7410         if (ioc->facts.MaxDevHandle % 8)
7411                 ioc->pd_handles_sz++;
7412         ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
7413             GFP_KERNEL);
7414         if (!ioc->pd_handles) {
7415                 r = -ENOMEM;
7416                 goto out_free_resources;
7417         }
7418         ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
7419             GFP_KERNEL);
7420         if (!ioc->blocking_handles) {
7421                 r = -ENOMEM;
7422                 goto out_free_resources;
7423         }
7424
7425         /* allocate memory for pending OS device add list */
7426         ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
7427         if (ioc->facts.MaxDevHandle % 8)
7428                 ioc->pend_os_device_add_sz++;
7429         ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
7430             GFP_KERNEL);
7431         if (!ioc->pend_os_device_add) {
7432                 r = -ENOMEM;
7433                 goto out_free_resources;
7434         }
7435
7436         ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
7437         ioc->device_remove_in_progress =
7438                 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
7439         if (!ioc->device_remove_in_progress) {
7440                 r = -ENOMEM;
7441                 goto out_free_resources;
7442         }
7443
7444         ioc->fwfault_debug = mpt3sas_fwfault_debug;
7445
7446         /* base internal command bits */
7447         mutex_init(&ioc->base_cmds.mutex);
7448         ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7449         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
7450
7451         /* port_enable command bits */
7452         ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7453         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
7454
7455         /* transport internal command bits */
7456         ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7457         ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
7458         mutex_init(&ioc->transport_cmds.mutex);
7459
7460         /* scsih internal command bits */
7461         ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7462         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
7463         mutex_init(&ioc->scsih_cmds.mutex);
7464
7465         /* task management internal command bits */
7466         ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7467         ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
7468         mutex_init(&ioc->tm_cmds.mutex);
7469
7470         /* config page internal command bits */
7471         ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7472         ioc->config_cmds.status = MPT3_CMD_NOT_USED;
7473         mutex_init(&ioc->config_cmds.mutex);
7474
7475         /* ctl module internal command bits */
7476         ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
7477         ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
7478         ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
7479         mutex_init(&ioc->ctl_cmds.mutex);
7480
7481         if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
7482             !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
7483             !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
7484             !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
7485                 r = -ENOMEM;
7486                 goto out_free_resources;
7487         }
7488
7489         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
7490                 ioc->event_masks[i] = -1;
7491
7492         /* here we enable the events we care about */
7493         _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
7494         _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
7495         _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
7496         _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
7497         _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
7498         _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
7499         _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
7500         _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
7501         _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
7502         _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
7503         _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
7504         _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
7505         _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
7506         if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
7507                 if (ioc->is_gen35_ioc) {
7508                         _base_unmask_events(ioc,
7509                                 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
7510                         _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);
7511                         _base_unmask_events(ioc,
7512                                 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
7513                 }
7514         }
7515         r = _base_make_ioc_operational(ioc);
7516         if (r)
7517                 goto out_free_resources;
7518
7519         /*
7520          * Copy current copy of IOCFacts in prev_fw_facts
7521          * and it will be used during online firmware upgrade.
7522          */
7523         memcpy(&ioc->prev_fw_facts, &ioc->facts,
7524             sizeof(struct mpt3sas_facts));
7525
7526         ioc->non_operational_loop = 0;
7527         ioc->ioc_coredump_loop = 0;
7528         ioc->got_task_abort_from_ioctl = 0;
7529         return 0;
7530
7531  out_free_resources:
7532
7533         ioc->remove_host = 1;
7534
7535         mpt3sas_base_free_resources(ioc);
7536         _base_release_memory_pools(ioc);
7537         pci_set_drvdata(ioc->pdev, NULL);
7538         kfree(ioc->cpu_msix_table);
7539         if (ioc->is_warpdrive)
7540                 kfree(ioc->reply_post_host_index);
7541         kfree(ioc->pd_handles);
7542         kfree(ioc->blocking_handles);
7543         kfree(ioc->device_remove_in_progress);
7544         kfree(ioc->pend_os_device_add);
7545         kfree(ioc->tm_cmds.reply);
7546         kfree(ioc->transport_cmds.reply);
7547         kfree(ioc->scsih_cmds.reply);
7548         kfree(ioc->config_cmds.reply);
7549         kfree(ioc->base_cmds.reply);
7550         kfree(ioc->port_enable_cmds.reply);
7551         kfree(ioc->ctl_cmds.reply);
7552         kfree(ioc->ctl_cmds.sense);
7553         kfree(ioc->pfacts);
7554         ioc->ctl_cmds.reply = NULL;
7555         ioc->base_cmds.reply = NULL;
7556         ioc->tm_cmds.reply = NULL;
7557         ioc->scsih_cmds.reply = NULL;
7558         ioc->transport_cmds.reply = NULL;
7559         ioc->config_cmds.reply = NULL;
7560         ioc->pfacts = NULL;
7561         return r;
7562 }
7563
7564
7565 /**
7566  * mpt3sas_base_detach - remove controller instance
7567  * @ioc: per adapter object
7568  */
7569 void
7570 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
7571 {
7572         dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7573
7574         mpt3sas_base_stop_watchdog(ioc);
7575         mpt3sas_base_free_resources(ioc);
7576         _base_release_memory_pools(ioc);
7577         mpt3sas_free_enclosure_list(ioc);
7578         pci_set_drvdata(ioc->pdev, NULL);
7579         kfree(ioc->cpu_msix_table);
7580         if (ioc->is_warpdrive)
7581                 kfree(ioc->reply_post_host_index);
7582         kfree(ioc->pd_handles);
7583         kfree(ioc->blocking_handles);
7584         kfree(ioc->device_remove_in_progress);
7585         kfree(ioc->pend_os_device_add);
7586         kfree(ioc->pfacts);
7587         kfree(ioc->ctl_cmds.reply);
7588         kfree(ioc->ctl_cmds.sense);
7589         kfree(ioc->base_cmds.reply);
7590         kfree(ioc->port_enable_cmds.reply);
7591         kfree(ioc->tm_cmds.reply);
7592         kfree(ioc->transport_cmds.reply);
7593         kfree(ioc->scsih_cmds.reply);
7594         kfree(ioc->config_cmds.reply);
7595 }
7596
7597 /**
7598  * _base_pre_reset_handler - pre reset handler
7599  * @ioc: per adapter object
7600  */
7601 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
7602 {
7603         mpt3sas_scsih_pre_reset_handler(ioc);
7604         mpt3sas_ctl_pre_reset_handler(ioc);
7605         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));
7606 }
7607
7608 /**
7609  * _base_clear_outstanding_mpt_commands - clears outstanding mpt commands
7610  * @ioc: per adapter object
7611  */
7612 static void
7613 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc)
7614 {
7615         dtmprintk(ioc,
7616             ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__));
7617         if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
7618                 ioc->transport_cmds.status |= MPT3_CMD_RESET;
7619                 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
7620                 complete(&ioc->transport_cmds.done);
7621         }
7622         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
7623                 ioc->base_cmds.status |= MPT3_CMD_RESET;
7624                 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
7625                 complete(&ioc->base_cmds.done);
7626         }
7627         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
7628                 ioc->port_enable_failed = 1;
7629                 ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
7630                 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
7631                 if (ioc->is_driver_loading) {
7632                         ioc->start_scan_failed =
7633                                 MPI2_IOCSTATUS_INTERNAL_ERROR;
7634                         ioc->start_scan = 0;
7635                         ioc->port_enable_cmds.status =
7636                                 MPT3_CMD_NOT_USED;
7637                 } else {
7638                         complete(&ioc->port_enable_cmds.done);
7639                 }
7640         }
7641         if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
7642                 ioc->config_cmds.status |= MPT3_CMD_RESET;
7643                 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
7644                 ioc->config_cmds.smid = USHRT_MAX;
7645                 complete(&ioc->config_cmds.done);
7646         }
7647 }
7648
7649 /**
7650  * _base_clear_outstanding_commands - clear all outstanding commands
7651  * @ioc: per adapter object
7652  */
7653 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc)
7654 {
7655         mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc);
7656         mpt3sas_ctl_clear_outstanding_ioctls(ioc);
7657         _base_clear_outstanding_mpt_commands(ioc);
7658 }
7659
7660 /**
7661  * _base_reset_done_handler - reset done handler
7662  * @ioc: per adapter object
7663  */
7664 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
7665 {
7666         mpt3sas_scsih_reset_done_handler(ioc);
7667         mpt3sas_ctl_reset_done_handler(ioc);
7668         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));
7669 }
7670
7671 /**
7672  * mpt3sas_wait_for_commands_to_complete - reset controller
7673  * @ioc: Pointer to MPT_ADAPTER structure
7674  *
7675  * This function is waiting 10s for all pending commands to complete
7676  * prior to putting controller in reset.
7677  */
7678 void
7679 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
7680 {
7681         u32 ioc_state;
7682
7683         ioc->pending_io_count = 0;
7684
7685         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
7686         if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
7687                 return;
7688
7689         /* pending command count */
7690         ioc->pending_io_count = scsi_host_busy(ioc->shost);
7691
7692         if (!ioc->pending_io_count)
7693                 return;
7694
7695         /* wait for pending commands to complete */
7696         wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
7697 }
7698
7699 /**
7700  * _base_check_ioc_facts_changes - Look for increase/decrease of IOCFacts
7701  *     attributes during online firmware upgrade and update the corresponding
7702  *     IOC variables accordingly.
7703  *
7704  * @ioc: Pointer to MPT_ADAPTER structure
7705  */
7706 static int
7707 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc)
7708 {
7709         u16 pd_handles_sz;
7710         void *pd_handles = NULL, *blocking_handles = NULL;
7711         void *pend_os_device_add = NULL, *device_remove_in_progress = NULL;
7712         struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts;
7713
7714         if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) {
7715                 pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
7716                 if (ioc->facts.MaxDevHandle % 8)
7717                         pd_handles_sz++;
7718
7719                 pd_handles = krealloc(ioc->pd_handles, pd_handles_sz,
7720                     GFP_KERNEL);
7721                 if (!pd_handles) {
7722                         ioc_info(ioc,
7723                             "Unable to allocate the memory for pd_handles of sz: %d\n",
7724                             pd_handles_sz);
7725                         return -ENOMEM;
7726                 }
7727                 memset(pd_handles + ioc->pd_handles_sz, 0,
7728                     (pd_handles_sz - ioc->pd_handles_sz));
7729                 ioc->pd_handles = pd_handles;
7730
7731                 blocking_handles = krealloc(ioc->blocking_handles,
7732                     pd_handles_sz, GFP_KERNEL);
7733                 if (!blocking_handles) {
7734                         ioc_info(ioc,
7735                             "Unable to allocate the memory for "
7736                             "blocking_handles of sz: %d\n",
7737                             pd_handles_sz);
7738                         return -ENOMEM;
7739                 }
7740                 memset(blocking_handles + ioc->pd_handles_sz, 0,
7741                     (pd_handles_sz - ioc->pd_handles_sz));
7742                 ioc->blocking_handles = blocking_handles;
7743                 ioc->pd_handles_sz = pd_handles_sz;
7744
7745                 pend_os_device_add = krealloc(ioc->pend_os_device_add,
7746                     pd_handles_sz, GFP_KERNEL);
7747                 if (!pend_os_device_add) {
7748                         ioc_info(ioc,
7749                             "Unable to allocate the memory for pend_os_device_add of sz: %d\n",
7750                             pd_handles_sz);
7751                         return -ENOMEM;
7752                 }
7753                 memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0,
7754                     (pd_handles_sz - ioc->pend_os_device_add_sz));
7755                 ioc->pend_os_device_add = pend_os_device_add;
7756                 ioc->pend_os_device_add_sz = pd_handles_sz;
7757
7758                 device_remove_in_progress = krealloc(
7759                     ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL);
7760                 if (!device_remove_in_progress) {
7761                         ioc_info(ioc,
7762                             "Unable to allocate the memory for "
7763                             "device_remove_in_progress of sz: %d\n "
7764                             , pd_handles_sz);
7765                         return -ENOMEM;
7766                 }
7767                 memset(device_remove_in_progress +
7768                     ioc->device_remove_in_progress_sz, 0,
7769                     (pd_handles_sz - ioc->device_remove_in_progress_sz));
7770                 ioc->device_remove_in_progress = device_remove_in_progress;
7771                 ioc->device_remove_in_progress_sz = pd_handles_sz;
7772         }
7773
7774         memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts));
7775         return 0;
7776 }
7777
7778 /**
7779  * mpt3sas_base_hard_reset_handler - reset controller
7780  * @ioc: Pointer to MPT_ADAPTER structure
7781  * @type: FORCE_BIG_HAMMER or SOFT_RESET
7782  *
7783  * Return: 0 for success, non-zero for failure.
7784  */
7785 int
7786 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
7787         enum reset_type type)
7788 {
7789         int r;
7790         unsigned long flags;
7791         u32 ioc_state;
7792         u8 is_fault = 0, is_trigger = 0;
7793
7794         dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__));
7795
7796         if (ioc->pci_error_recovery) {
7797                 ioc_err(ioc, "%s: pci error recovery reset\n", __func__);
7798                 r = 0;
7799                 goto out_unlocked;
7800         }
7801
7802         if (mpt3sas_fwfault_debug)
7803                 mpt3sas_halt_firmware(ioc);
7804
7805         /* wait for an active reset in progress to complete */
7806         mutex_lock(&ioc->reset_in_progress_mutex);
7807
7808         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
7809         ioc->shost_recovery = 1;
7810         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
7811
7812         if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
7813             MPT3_DIAG_BUFFER_IS_REGISTERED) &&
7814             (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
7815             MPT3_DIAG_BUFFER_IS_RELEASED))) {
7816                 is_trigger = 1;
7817                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
7818                 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT ||
7819                     (ioc_state & MPI2_IOC_STATE_MASK) ==
7820                     MPI2_IOC_STATE_COREDUMP)
7821                         is_fault = 1;
7822         }
7823         _base_pre_reset_handler(ioc);
7824         mpt3sas_wait_for_commands_to_complete(ioc);
7825         mpt3sas_base_mask_interrupts(ioc);
7826         r = _base_make_ioc_ready(ioc, type);
7827         if (r)
7828                 goto out;
7829         _base_clear_outstanding_commands(ioc);
7830
7831         /* If this hard reset is called while port enable is active, then
7832          * there is no reason to call make_ioc_operational
7833          */
7834         if (ioc->is_driver_loading && ioc->port_enable_failed) {
7835                 ioc->remove_host = 1;
7836                 r = -EFAULT;
7837                 goto out;
7838         }
7839         r = _base_get_ioc_facts(ioc);
7840         if (r)
7841                 goto out;
7842
7843         r = _base_check_ioc_facts_changes(ioc);
7844         if (r) {
7845                 ioc_info(ioc,
7846                     "Some of the parameters got changed in this new firmware"
7847                     " image and it requires system reboot\n");
7848                 goto out;
7849         }
7850         if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
7851                 panic("%s: Issue occurred with flashing controller firmware."
7852                       "Please reboot the system and ensure that the correct"
7853                       " firmware version is running\n", ioc->name);
7854
7855         r = _base_make_ioc_operational(ioc);
7856         if (!r)
7857                 _base_reset_done_handler(ioc);
7858
7859  out:
7860         ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED");
7861
7862         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
7863         ioc->shost_recovery = 0;
7864         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
7865         ioc->ioc_reset_count++;
7866         mutex_unlock(&ioc->reset_in_progress_mutex);
7867
7868  out_unlocked:
7869         if ((r == 0) && is_trigger) {
7870                 if (is_fault)
7871                         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
7872                 else
7873                         mpt3sas_trigger_master(ioc,
7874                             MASTER_TRIGGER_ADAPTER_RESET);
7875         }
7876         dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__));
7877         return r;
7878 }