drivers/scsi/smartpqi/smartpqi_init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *    driver for Microchip PQI-based storage controllers
   4  *    Copyright (c) 2019-2023 Microchip Technology Inc. and its subsidiaries
   5  *    Copyright (c) 2016-2018 Microsemi Corporation
   6  *    Copyright (c) 2016 PMC-Sierra, Inc.
   7  *
   8  *    Questions/Comments/Bugfixes to storagedev@microchip.com
   9  *
  10  */
  11
  12 #include <linux/module.h>
  13 #include <linux/kernel.h>
  14 #include <linux/pci.h>
  15 #include <linux/delay.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/sched.h>
  18 #include <linux/rtc.h>
  19 #include <linux/bcd.h>
  20 #include <linux/reboot.h>
  21 #include <linux/cciss_ioctl.h>
  22 #include <linux/blk-mq-pci.h>
  23 #include <scsi/scsi_host.h>
  24 #include <scsi/scsi_cmnd.h>
  25 #include <scsi/scsi_device.h>
  26 #include <scsi/scsi_eh.h>
  27 #include <scsi/scsi_transport_sas.h>
  28 #include <asm/unaligned.h>
  29 #include "smartpqi.h"
  30 #include "smartpqi_sis.h"
  31
  32 #if !defined(BUILD_TIMESTAMP)
  33 #define BUILD_TIMESTAMP
  34 #endif
  35
  36 #define DRIVER_VERSION          "2.1.26-030"
  37 #define DRIVER_MAJOR            2
  38 #define DRIVER_MINOR            1
  39 #define DRIVER_RELEASE          26
  40 #define DRIVER_REVISION         30
  41
  42 #define DRIVER_NAME             "Microchip SmartPQI Driver (v" \
  43                                 DRIVER_VERSION BUILD_TIMESTAMP ")"
  44 #define DRIVER_NAME_SHORT       "smartpqi"
  45
  46 #define PQI_EXTRA_SGL_MEMORY    (12 * sizeof(struct pqi_sg_descriptor))
  47
  48 #define PQI_POST_RESET_DELAY_SECS                       5
  49 #define PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS      10
  50
  51 #define PQI_NO_COMPLETION       ((void *)-1)
  52
  53 MODULE_AUTHOR("Microchip");
  54 MODULE_DESCRIPTION("Driver for Microchip Smart Family Controller version "
  55         DRIVER_VERSION);
  56 MODULE_VERSION(DRIVER_VERSION);
  57 MODULE_LICENSE("GPL");
  58
  59 struct pqi_cmd_priv {
  60         int this_residual;
  61 };
  62
  63 static struct pqi_cmd_priv *pqi_cmd_priv(struct scsi_cmnd *cmd)
  64 {
  65         return scsi_cmd_priv(cmd);
  66 }
  67
  68 static void pqi_verify_structures(void);
  69 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,
  70         enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason);
  71 static void pqi_ctrl_offline_worker(struct work_struct *work);
  72 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
  73 static void pqi_scan_start(struct Scsi_Host *shost);
  74 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
  75         struct pqi_queue_group *queue_group, enum pqi_io_path path,
  76         struct pqi_io_request *io_request);
  77 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
  78         struct pqi_iu_header *request, unsigned int flags,
  79         struct pqi_raid_error_info *error_info);
  80 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
  81         struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
  82         unsigned int cdb_length, struct pqi_queue_group *queue_group,
  83         struct pqi_encryption_info *encryption_info, bool raid_bypass, bool io_high_prio);
  84 static  int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
  85         struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
  86         struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
  87         struct pqi_scsi_dev_raid_map_data *rmd);
  88 static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
  89         struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
  90         struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
  91         struct pqi_scsi_dev_raid_map_data *rmd);
  92 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
  93 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
  94 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs);
  95 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info);
  96 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
  97 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
  98 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
  99         struct pqi_scsi_dev *device, u8 lun, unsigned long timeout_msecs);
 100 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info);
 101 static void pqi_tmf_worker(struct work_struct *work);
 102
 103 /* for flags argument to pqi_submit_raid_request_synchronous() */
 104 #define PQI_SYNC_FLAGS_INTERRUPTABLE    0x1
 105
 106 static struct scsi_transport_template *pqi_sas_transport_template;
 107
 108 static atomic_t pqi_controller_count = ATOMIC_INIT(0);
 109
 110 enum pqi_lockup_action {
 111         NONE,
 112         REBOOT,
 113         PANIC
 114 };
 115
 116 static enum pqi_lockup_action pqi_lockup_action = NONE;
 117
 118 static struct {
 119         enum pqi_lockup_action  action;
 120         char                    *name;
 121 } pqi_lockup_actions[] = {
 122         {
 123                 .action = NONE,
 124                 .name = "none",
 125         },
 126         {
 127                 .action = REBOOT,
 128                 .name = "reboot",
 129         },
 130         {
 131                 .action = PANIC,
 132                 .name = "panic",
 133         },
 134 };
 135
 136 static unsigned int pqi_supported_event_types[] = {
 137         PQI_EVENT_TYPE_HOTPLUG,
 138         PQI_EVENT_TYPE_HARDWARE,
 139         PQI_EVENT_TYPE_PHYSICAL_DEVICE,
 140         PQI_EVENT_TYPE_LOGICAL_DEVICE,
 141         PQI_EVENT_TYPE_OFA,
 142         PQI_EVENT_TYPE_AIO_STATE_CHANGE,
 143         PQI_EVENT_TYPE_AIO_CONFIG_CHANGE,
 144 };
 145
 146 static int pqi_disable_device_id_wildcards;
 147 module_param_named(disable_device_id_wildcards,
 148         pqi_disable_device_id_wildcards, int, 0644);
 149 MODULE_PARM_DESC(disable_device_id_wildcards,
 150         "Disable device ID wildcards.");
 151
 152 static int pqi_disable_heartbeat;
 153 module_param_named(disable_heartbeat,
 154         pqi_disable_heartbeat, int, 0644);
 155 MODULE_PARM_DESC(disable_heartbeat,
 156         "Disable heartbeat.");
 157
 158 static int pqi_disable_ctrl_shutdown;
 159 module_param_named(disable_ctrl_shutdown,
 160         pqi_disable_ctrl_shutdown, int, 0644);
 161 MODULE_PARM_DESC(disable_ctrl_shutdown,
 162         "Disable controller shutdown when controller locked up.");
 163
 164 static char *pqi_lockup_action_param;
 165 module_param_named(lockup_action,
 166         pqi_lockup_action_param, charp, 0644);
 167 MODULE_PARM_DESC(lockup_action, "Action to take when controller locked up.\n"
 168         "\t\tSupported: none, reboot, panic\n"
 169         "\t\tDefault: none");
 170
 171 static int pqi_expose_ld_first;
 172 module_param_named(expose_ld_first,
 173         pqi_expose_ld_first, int, 0644);
 174 MODULE_PARM_DESC(expose_ld_first, "Expose logical drives before physical drives.");
 175
 176 static int pqi_hide_vsep;
 177 module_param_named(hide_vsep,
 178         pqi_hide_vsep, int, 0644);
 179 MODULE_PARM_DESC(hide_vsep, "Hide the virtual SEP for direct attached drives.");
 180
 181 static int pqi_disable_managed_interrupts;
 182 module_param_named(disable_managed_interrupts,
 183         pqi_disable_managed_interrupts, int, 0644);
 184 MODULE_PARM_DESC(disable_managed_interrupts,
 185         "Disable the kernel automatically assigning SMP affinity to IRQs.");
 186
 187 static unsigned int pqi_ctrl_ready_timeout_secs;
 188 module_param_named(ctrl_ready_timeout,
 189         pqi_ctrl_ready_timeout_secs, uint, 0644);
 190 MODULE_PARM_DESC(ctrl_ready_timeout,
 191         "Timeout in seconds for driver to wait for controller ready.");
 192
 193 static char *raid_levels[] = {
 194         "RAID-0",
 195         "RAID-4",
 196         "RAID-1(1+0)",
 197         "RAID-5",
 198         "RAID-5+1",
 199         "RAID-6",
 200         "RAID-1(Triple)",
 201 };
 202
 203 static char *pqi_raid_level_to_string(u8 raid_level)
 204 {
 205         if (raid_level < ARRAY_SIZE(raid_levels))
 206                 return raid_levels[raid_level];
 207
 208         return "RAID UNKNOWN";
 209 }
 210
 211 #define SA_RAID_0               0
 212 #define SA_RAID_4               1
 213 #define SA_RAID_1               2       /* also used for RAID 10 */
 214 #define SA_RAID_5               3       /* also used for RAID 50 */
 215 #define SA_RAID_51              4
 216 #define SA_RAID_6               5       /* also used for RAID 60 */
 217 #define SA_RAID_TRIPLE          6       /* also used for RAID 1+0 Triple */
 218 #define SA_RAID_MAX             SA_RAID_TRIPLE
 219 #define SA_RAID_UNKNOWN         0xff
 220
 221 static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
 222 {
 223         pqi_prep_for_scsi_done(scmd);
 224         scsi_done(scmd);
 225 }
 226
 227 static inline void pqi_disable_write_same(struct scsi_device *sdev)
 228 {
 229         sdev->no_write_same = 1;
 230 }
 231
 232 static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
 233 {
 234         return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
 235 }
 236
 237 static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
 238 {
 239         return !device->is_physical_device;
 240 }
 241
 242 static inline bool pqi_is_external_raid_addr(u8 *scsi3addr)
 243 {
 244         return scsi3addr[2] != 0;
 245 }
 246
 247 static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
 248 {
 249         return !ctrl_info->controller_online;
 250 }
 251
 252 static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
 253 {
 254         if (ctrl_info->controller_online)
 255                 if (!sis_is_firmware_running(ctrl_info))
 256                         pqi_take_ctrl_offline(ctrl_info, PQI_FIRMWARE_KERNEL_NOT_UP);
 257 }
 258
 259 static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
 260 {
 261         return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
 262 }
 263
 264 #define PQI_DRIVER_SCRATCH_PQI_MODE                     0x1
 265 #define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED          0x2
 266
 267 static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
 268 {
 269         return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
 270 }
 271
 272 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
 273         enum pqi_ctrl_mode mode)
 274 {
 275         u32 driver_scratch;
 276
 277         driver_scratch = sis_read_driver_scratch(ctrl_info);
 278
 279         if (mode == PQI_MODE)
 280                 driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
 281         else
 282                 driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
 283
 284         sis_write_driver_scratch(ctrl_info, driver_scratch);
 285 }
 286
 287 static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
 288 {
 289         return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
 290 }
 291
 292 static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
 293 {
 294         u32 driver_scratch;
 295
 296         driver_scratch = sis_read_driver_scratch(ctrl_info);
 297
 298         if (is_supported)
 299                 driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
 300         else
 301                 driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
 302
 303         sis_write_driver_scratch(ctrl_info, driver_scratch);
 304 }
 305
 306 static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
 307 {
 308         ctrl_info->scan_blocked = true;
 309         mutex_lock(&ctrl_info->scan_mutex);
 310 }
 311
 312 static inline void pqi_ctrl_unblock_scan(struct pqi_ctrl_info *ctrl_info)
 313 {
 314         ctrl_info->scan_blocked = false;
 315         mutex_unlock(&ctrl_info->scan_mutex);
 316 }
 317
 318 static inline bool pqi_ctrl_scan_blocked(struct pqi_ctrl_info *ctrl_info)
 319 {
 320         return ctrl_info->scan_blocked;
 321 }
 322
 323 static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)
 324 {
 325         mutex_lock(&ctrl_info->lun_reset_mutex);
 326 }
 327
 328 static inline void pqi_ctrl_unblock_device_reset(struct pqi_ctrl_info *ctrl_info)
 329 {
 330         mutex_unlock(&ctrl_info->lun_reset_mutex);
 331 }
 332
 333 static inline void pqi_scsi_block_requests(struct pqi_ctrl_info *ctrl_info)
 334 {
 335         struct Scsi_Host *shost;
 336         unsigned int num_loops;
 337         int msecs_sleep;
 338
 339         shost = ctrl_info->scsi_host;
 340
 341         scsi_block_requests(shost);
 342
 343         num_loops = 0;
 344         msecs_sleep = 20;
 345         while (scsi_host_busy(shost)) {
 346                 num_loops++;
 347                 if (num_loops == 10)
 348                         msecs_sleep = 500;
 349                 msleep(msecs_sleep);
 350         }
 351 }
 352
 353 static inline void pqi_scsi_unblock_requests(struct pqi_ctrl_info *ctrl_info)
 354 {
 355         scsi_unblock_requests(ctrl_info->scsi_host);
 356 }
 357
 358 static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)
 359 {
 360         atomic_inc(&ctrl_info->num_busy_threads);
 361 }
 362
 363 static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)
 364 {
 365         atomic_dec(&ctrl_info->num_busy_threads);
 366 }
 367
 368 static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
 369 {
 370         return ctrl_info->block_requests;
 371 }
 372
 373 static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)
 374 {
 375         ctrl_info->block_requests = true;
 376 }
 377
 378 static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)
 379 {
 380         ctrl_info->block_requests = false;
 381         wake_up_all(&ctrl_info->block_requests_wait);
 382 }
 383
 384 static void pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
 385 {
 386         if (!pqi_ctrl_blocked(ctrl_info))
 387                 return;
 388
 389         atomic_inc(&ctrl_info->num_blocked_threads);
 390         wait_event(ctrl_info->block_requests_wait,
 391                 !pqi_ctrl_blocked(ctrl_info));
 392         atomic_dec(&ctrl_info->num_blocked_threads);
 393 }
 394
 395 #define PQI_QUIESCE_WARNING_TIMEOUT_SECS                10
 396
 397 static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)
 398 {
 399         unsigned long start_jiffies;
 400         unsigned long warning_timeout;
 401         bool displayed_warning;
 402
 403         displayed_warning = false;
 404         start_jiffies = jiffies;
 405         warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
 406
 407         while (atomic_read(&ctrl_info->num_busy_threads) >
 408                 atomic_read(&ctrl_info->num_blocked_threads)) {
 409                 if (time_after(jiffies, warning_timeout)) {
 410                         dev_warn(&ctrl_info->pci_dev->dev,
 411                                 "waiting %u seconds for driver activity to quiesce\n",
 412                                 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
 413                         displayed_warning = true;
 414                         warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * HZ) + jiffies;
 415                 }
 416                 usleep_range(1000, 2000);
 417         }
 418
 419         if (displayed_warning)
 420                 dev_warn(&ctrl_info->pci_dev->dev,
 421                         "driver activity quiesced after waiting for %u seconds\n",
 422                         jiffies_to_msecs(jiffies - start_jiffies) / 1000);
 423 }
 424
 425 static inline bool pqi_device_offline(struct pqi_scsi_dev *device)
 426 {
 427         return device->device_offline;
 428 }
 429
 430 static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)
 431 {
 432         mutex_lock(&ctrl_info->ofa_mutex);
 433 }
 434
 435 static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)
 436 {
 437         mutex_unlock(&ctrl_info->ofa_mutex);
 438 }
 439
 440 static inline void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)
 441 {
 442         mutex_lock(&ctrl_info->ofa_mutex);
 443         mutex_unlock(&ctrl_info->ofa_mutex);
 444 }
 445
 446 static inline bool pqi_ofa_in_progress(struct pqi_ctrl_info *ctrl_info)
 447 {
 448         return mutex_is_locked(&ctrl_info->ofa_mutex);
 449 }
 450
 451 static inline void pqi_device_remove_start(struct pqi_scsi_dev *device)
 452 {
 453         device->in_remove = true;
 454 }
 455
 456 static inline bool pqi_device_in_remove(struct pqi_scsi_dev *device)
 457 {
 458         return device->in_remove;
 459 }
 460
 461 static inline void pqi_device_reset_start(struct pqi_scsi_dev *device, u8 lun)
 462 {
 463         device->in_reset[lun] = true;
 464 }
 465
 466 static inline void pqi_device_reset_done(struct pqi_scsi_dev *device, u8 lun)
 467 {
 468         device->in_reset[lun] = false;
 469 }
 470
 471 static inline bool pqi_device_in_reset(struct pqi_scsi_dev *device, u8 lun)
 472 {
 473         return device->in_reset[lun];
 474 }
 475
 476 static inline int pqi_event_type_to_event_index(unsigned int event_type)
 477 {
 478         int index;
 479
 480         for (index = 0; index < ARRAY_SIZE(pqi_supported_event_types); index++)
 481                 if (event_type == pqi_supported_event_types[index])
 482                         return index;
 483
 484         return -1;
 485 }
 486
 487 static inline bool pqi_is_supported_event(unsigned int event_type)
 488 {
 489         return pqi_event_type_to_event_index(event_type) != -1;
 490 }
 491
 492 static inline void pqi_schedule_rescan_worker_with_delay(struct pqi_ctrl_info *ctrl_info,
 493         unsigned long delay)
 494 {
 495         if (pqi_ctrl_offline(ctrl_info))
 496                 return;
 497
 498         schedule_delayed_work(&ctrl_info->rescan_work, delay);
 499 }
 500
 501 static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
 502 {
 503         pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);
 504 }
 505
 506 #define PQI_RESCAN_WORK_DELAY   (10 * HZ)
 507
 508 static inline void pqi_schedule_rescan_worker_delayed(struct pqi_ctrl_info *ctrl_info)
 509 {
 510         pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);
 511 }
 512
 513 static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)
 514 {
 515         cancel_delayed_work_sync(&ctrl_info->rescan_work);
 516 }
 517
 518 static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)
 519 {
 520         if (!ctrl_info->heartbeat_counter)
 521                 return 0;
 522
 523         return readl(ctrl_info->heartbeat_counter);
 524 }
 525
 526 static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
 527 {
 528         return readb(ctrl_info->soft_reset_status);
 529 }
 530
 531 static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
 532 {
 533         u8 status;
 534
 535         status = pqi_read_soft_reset_status(ctrl_info);
 536         status &= ~PQI_SOFT_RESET_ABORT;
 537         writeb(status, ctrl_info->soft_reset_status);
 538 }
 539
 540 static inline bool pqi_is_io_high_priority(struct pqi_scsi_dev *device, struct scsi_cmnd *scmd)
 541 {
 542         bool io_high_prio;
 543         int priority_class;
 544
 545         io_high_prio = false;
 546
 547         if (device->ncq_prio_enable) {
 548                 priority_class =
 549                         IOPRIO_PRIO_CLASS(req_get_ioprio(scsi_cmd_to_rq(scmd)));
 550                 if (priority_class == IOPRIO_CLASS_RT) {
 551                         /* Set NCQ priority for read/write commands. */
 552                         switch (scmd->cmnd[0]) {
 553                         case WRITE_16:
 554                         case READ_16:
 555                         case WRITE_12:
 556                         case READ_12:
 557                         case WRITE_10:
 558                         case READ_10:
 559                         case WRITE_6:
 560                         case READ_6:
 561                                 io_high_prio = true;
 562                                 break;
 563                         }
 564                 }
 565         }
 566
 567         return io_high_prio;
 568 }
 569
 570 static int pqi_map_single(struct pci_dev *pci_dev,
 571         struct pqi_sg_descriptor *sg_descriptor, void *buffer,
 572         size_t buffer_length, enum dma_data_direction data_direction)
 573 {
 574         dma_addr_t bus_address;
 575
 576         if (!buffer || buffer_length == 0 || data_direction == DMA_NONE)
 577                 return 0;
 578
 579         bus_address = dma_map_single(&pci_dev->dev, buffer, buffer_length,
 580                 data_direction);
 581         if (dma_mapping_error(&pci_dev->dev, bus_address))
 582                 return -ENOMEM;
 583
 584         put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
 585         put_unaligned_le32(buffer_length, &sg_descriptor->length);
 586         put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
 587
 588         return 0;
 589 }
 590
 591 static void pqi_pci_unmap(struct pci_dev *pci_dev,
 592         struct pqi_sg_descriptor *descriptors, int num_descriptors,
 593         enum dma_data_direction data_direction)
 594 {
 595         int i;
 596
 597         if (data_direction == DMA_NONE)
 598                 return;
 599
 600         for (i = 0; i < num_descriptors; i++)
 601                 dma_unmap_single(&pci_dev->dev,
 602                         (dma_addr_t)get_unaligned_le64(&descriptors[i].address),
 603                         get_unaligned_le32(&descriptors[i].length),
 604                         data_direction);
 605 }
 606
 607 static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
 608         struct pqi_raid_path_request *request, u8 cmd,
 609         u8 *scsi3addr, void *buffer, size_t buffer_length,
 610         u16 vpd_page, enum dma_data_direction *dir)
 611 {
 612         u8 *cdb;
 613         size_t cdb_length = buffer_length;
 614
 615         memset(request, 0, sizeof(*request));
 616
 617         request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
 618         put_unaligned_le16(offsetof(struct pqi_raid_path_request,
 619                 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
 620                 &request->header.iu_length);
 621         put_unaligned_le32(buffer_length, &request->buffer_length);
 622         memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
 623         request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
 624         request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
 625
 626         cdb = request->cdb;
 627
 628         switch (cmd) {
 629         case INQUIRY:
 630                 request->data_direction = SOP_READ_FLAG;
 631                 cdb[0] = INQUIRY;
 632                 if (vpd_page & VPD_PAGE) {
 633                         cdb[1] = 0x1;
 634                         cdb[2] = (u8)vpd_page;
 635                 }
 636                 cdb[4] = (u8)cdb_length;
 637                 break;
 638         case CISS_REPORT_LOG:
 639         case CISS_REPORT_PHYS:
 640                 request->data_direction = SOP_READ_FLAG;
 641                 cdb[0] = cmd;
 642                 if (cmd == CISS_REPORT_PHYS) {
 643                         if (ctrl_info->rpl_extended_format_4_5_supported)
 644                                 cdb[1] = CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_4;
 645                         else
 646                                 cdb[1] = CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_2;
 647                 } else {
 648                         cdb[1] = ctrl_info->ciss_report_log_flags;
 649                 }
 650                 put_unaligned_be32(cdb_length, &cdb[6]);
 651                 break;
 652         case CISS_GET_RAID_MAP:
 653                 request->data_direction = SOP_READ_FLAG;
 654                 cdb[0] = CISS_READ;
 655                 cdb[1] = CISS_GET_RAID_MAP;
 656                 put_unaligned_be32(cdb_length, &cdb[6]);
 657                 break;
 658         case SA_FLUSH_CACHE:
 659                 request->header.driver_flags = PQI_DRIVER_NONBLOCKABLE_REQUEST;
 660                 request->data_direction = SOP_WRITE_FLAG;
 661                 cdb[0] = BMIC_WRITE;
 662                 cdb[6] = BMIC_FLUSH_CACHE;
 663                 put_unaligned_be16(cdb_length, &cdb[7]);
 664                 break;
 665         case BMIC_SENSE_DIAG_OPTIONS:
 666                 cdb_length = 0;
 667                 fallthrough;
 668         case BMIC_IDENTIFY_CONTROLLER:
 669         case BMIC_IDENTIFY_PHYSICAL_DEVICE:
 670         case BMIC_SENSE_SUBSYSTEM_INFORMATION:
 671         case BMIC_SENSE_FEATURE:
 672                 request->data_direction = SOP_READ_FLAG;
 673                 cdb[0] = BMIC_READ;
 674                 cdb[6] = cmd;
 675                 put_unaligned_be16(cdb_length, &cdb[7]);
 676                 break;
 677         case BMIC_SET_DIAG_OPTIONS:
 678                 cdb_length = 0;
 679                 fallthrough;
 680         case BMIC_WRITE_HOST_WELLNESS:
 681                 request->data_direction = SOP_WRITE_FLAG;
 682                 cdb[0] = BMIC_WRITE;
 683                 cdb[6] = cmd;
 684                 put_unaligned_be16(cdb_length, &cdb[7]);
 685                 break;
 686         case BMIC_CSMI_PASSTHRU:
 687                 request->data_direction = SOP_BIDIRECTIONAL;
 688                 cdb[0] = BMIC_WRITE;
 689                 cdb[5] = CSMI_CC_SAS_SMP_PASSTHRU;
 690                 cdb[6] = cmd;
 691                 put_unaligned_be16(cdb_length, &cdb[7]);
 692                 break;
 693         default:
 694                 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);
 695                 break;
 696         }
 697
 698         switch (request->data_direction) {
 699         case SOP_READ_FLAG:
 700                 *dir = DMA_FROM_DEVICE;
 701                 break;
 702         case SOP_WRITE_FLAG:
 703                 *dir = DMA_TO_DEVICE;
 704                 break;
 705         case SOP_NO_DIRECTION_FLAG:
 706                 *dir = DMA_NONE;
 707                 break;
 708         default:
 709                 *dir = DMA_BIDIRECTIONAL;
 710                 break;
 711         }
 712
 713         return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
 714                 buffer, buffer_length, *dir);
 715 }
 716
 717 static inline void pqi_reinit_io_request(struct pqi_io_request *io_request)
 718 {
 719         io_request->scmd = NULL;
 720         io_request->status = 0;
 721         io_request->error_info = NULL;
 722         io_request->raid_bypass = false;
 723 }
 724
 725 static inline struct pqi_io_request *pqi_alloc_io_request(struct pqi_ctrl_info *ctrl_info, struct scsi_cmnd *scmd)
 726 {
 727         struct pqi_io_request *io_request;
 728         u16 i;
 729
 730         if (scmd) { /* SML I/O request */
 731                 u32 blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
 732
 733                 i = blk_mq_unique_tag_to_tag(blk_tag);
 734                 io_request = &ctrl_info->io_request_pool[i];
 735                 if (atomic_inc_return(&io_request->refcount) > 1) {
 736                         atomic_dec(&io_request->refcount);
 737                         return NULL;
 738                 }
 739         } else { /* IOCTL or driver internal request */
 740                 /*
 741                  * benignly racy - may have to wait for an open slot.
 742                  * command slot range is scsi_ml_can_queue -
 743                  *         [scsi_ml_can_queue + (PQI_RESERVED_IO_SLOTS - 1)]
 744                  */
 745                 i = 0;
 746                 while (1) {
 747                         io_request = &ctrl_info->io_request_pool[ctrl_info->scsi_ml_can_queue + i];
 748                         if (atomic_inc_return(&io_request->refcount) == 1)
 749                                 break;
 750                         atomic_dec(&io_request->refcount);
 751                         i = (i + 1) % PQI_RESERVED_IO_SLOTS;
 752                 }
 753         }
 754
 755         if (io_request)
 756                 pqi_reinit_io_request(io_request);
 757
 758         return io_request;
 759 }
 760
 761 static void pqi_free_io_request(struct pqi_io_request *io_request)
 762 {
 763         atomic_dec(&io_request->refcount);
 764 }
 765
 766 static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,
 767         u8 *scsi3addr, void *buffer, size_t buffer_length, u16 vpd_page,
 768         struct pqi_raid_error_info *error_info)
 769 {
 770         int rc;
 771         struct pqi_raid_path_request request;
 772         enum dma_data_direction dir;
 773
 774         rc = pqi_build_raid_path_request(ctrl_info, &request, cmd, scsi3addr,
 775                 buffer, buffer_length, vpd_page, &dir);
 776         if (rc)
 777                 return rc;
 778
 779         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, error_info);
 780
 781         pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
 782
 783         return rc;
 784 }
 785
 786 /* helper functions for pqi_send_scsi_raid_request */
 787
 788 static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,
 789         u8 cmd, void *buffer, size_t buffer_length)
 790 {
 791         return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
 792                 buffer, buffer_length, 0, NULL);
 793 }
 794
 795 static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,
 796         u8 cmd, void *buffer, size_t buffer_length,
 797         struct pqi_raid_error_info *error_info)
 798 {
 799         return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
 800                 buffer, buffer_length, 0, error_info);
 801 }
 802
 803 static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
 804         struct bmic_identify_controller *buffer)
 805 {
 806         return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,
 807                 buffer, sizeof(*buffer));
 808 }
 809
 810 static inline int pqi_sense_subsystem_info(struct  pqi_ctrl_info *ctrl_info,
 811         struct bmic_sense_subsystem_info *sense_info)
 812 {
 813         return pqi_send_ctrl_raid_request(ctrl_info,
 814                 BMIC_SENSE_SUBSYSTEM_INFORMATION, sense_info,
 815                 sizeof(*sense_info));
 816 }
 817
 818 static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
 819         u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
 820 {
 821         return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,
 822                 buffer, buffer_length, vpd_page, NULL);
 823 }
 824
 825 static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
 826         struct pqi_scsi_dev *device,
 827         struct bmic_identify_physical_device *buffer, size_t buffer_length)
 828 {
 829         int rc;
 830         enum dma_data_direction dir;
 831         u16 bmic_device_index;
 832         struct pqi_raid_path_request request;
 833
 834         rc = pqi_build_raid_path_request(ctrl_info, &request,
 835                 BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
 836                 buffer_length, 0, &dir);
 837         if (rc)
 838                 return rc;
 839
 840         bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
 841         request.cdb[2] = (u8)bmic_device_index;
 842         request.cdb[9] = (u8)(bmic_device_index >> 8);
 843
 844         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
 845
 846         pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
 847
 848         return rc;
 849 }
 850
 851 static inline u32 pqi_aio_limit_to_bytes(__le16 *limit)
 852 {
 853         u32 bytes;
 854
 855         bytes = get_unaligned_le16(limit);
 856         if (bytes == 0)
 857                 bytes = ~0;
 858         else
 859                 bytes *= 1024;
 860
 861         return bytes;
 862 }
 863
 864 #pragma pack(1)
 865
 866 struct bmic_sense_feature_buffer {
 867         struct bmic_sense_feature_buffer_header header;
 868         struct bmic_sense_feature_io_page_aio_subpage aio_subpage;
 869 };
 870
 871 #pragma pack()
 872
 873 #define MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH       \
 874         offsetofend(struct bmic_sense_feature_buffer, \
 875                 aio_subpage.max_write_raid_1_10_3drive)
 876
 877 #define MINIMUM_AIO_SUBPAGE_LENGTH      \
 878         (offsetofend(struct bmic_sense_feature_io_page_aio_subpage, \
 879                 max_write_raid_1_10_3drive) - \
 880                 sizeof_field(struct bmic_sense_feature_io_page_aio_subpage, header))
 881
 882 static int pqi_get_advanced_raid_bypass_config(struct pqi_ctrl_info *ctrl_info)
 883 {
 884         int rc;
 885         enum dma_data_direction dir;
 886         struct pqi_raid_path_request request;
 887         struct bmic_sense_feature_buffer *buffer;
 888
 889         buffer = kmalloc(sizeof(*buffer), GFP_KERNEL);
 890         if (!buffer)
 891                 return -ENOMEM;
 892
 893         rc = pqi_build_raid_path_request(ctrl_info, &request, BMIC_SENSE_FEATURE, RAID_CTLR_LUNID,
 894                 buffer, sizeof(*buffer), 0, &dir);
 895         if (rc)
 896                 goto error;
 897
 898         request.cdb[2] = BMIC_SENSE_FEATURE_IO_PAGE;
 899         request.cdb[3] = BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE;
 900
 901         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
 902
 903         pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
 904
 905         if (rc)
 906                 goto error;
 907
 908         if (buffer->header.page_code != BMIC_SENSE_FEATURE_IO_PAGE ||
 909                 buffer->header.subpage_code !=
 910                         BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
 911                 get_unaligned_le16(&buffer->header.buffer_length) <
 912                         MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH ||
 913                 buffer->aio_subpage.header.page_code !=
 914                         BMIC_SENSE_FEATURE_IO_PAGE ||
 915                 buffer->aio_subpage.header.subpage_code !=
 916                         BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
 917                 get_unaligned_le16(&buffer->aio_subpage.header.page_length) <
 918                         MINIMUM_AIO_SUBPAGE_LENGTH) {
 919                 goto error;
 920         }
 921
 922         ctrl_info->max_transfer_encrypted_sas_sata =
 923                 pqi_aio_limit_to_bytes(
 924                         &buffer->aio_subpage.max_transfer_encrypted_sas_sata);
 925
 926         ctrl_info->max_transfer_encrypted_nvme =
 927                 pqi_aio_limit_to_bytes(
 928                         &buffer->aio_subpage.max_transfer_encrypted_nvme);
 929
 930         ctrl_info->max_write_raid_5_6 =
 931                 pqi_aio_limit_to_bytes(
 932                         &buffer->aio_subpage.max_write_raid_5_6);
 933
 934         ctrl_info->max_write_raid_1_10_2drive =
 935                 pqi_aio_limit_to_bytes(
 936                         &buffer->aio_subpage.max_write_raid_1_10_2drive);
 937
 938         ctrl_info->max_write_raid_1_10_3drive =
 939                 pqi_aio_limit_to_bytes(
 940                         &buffer->aio_subpage.max_write_raid_1_10_3drive);
 941
 942 error:
 943         kfree(buffer);
 944
 945         return rc;
 946 }
 947
 948 static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,
 949         enum bmic_flush_cache_shutdown_event shutdown_event)
 950 {
 951         int rc;
 952         struct bmic_flush_cache *flush_cache;
 953
 954         flush_cache = kzalloc(sizeof(*flush_cache), GFP_KERNEL);
 955         if (!flush_cache)
 956                 return -ENOMEM;
 957
 958         flush_cache->shutdown_event = shutdown_event;
 959
 960         rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,
 961                 sizeof(*flush_cache));
 962
 963         kfree(flush_cache);
 964
 965         return rc;
 966 }
 967
 968 int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,
 969         struct bmic_csmi_smp_passthru_buffer *buffer, size_t buffer_length,
 970         struct pqi_raid_error_info *error_info)
 971 {
 972         return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,
 973                 buffer, buffer_length, error_info);
 974 }
 975
 976 #define PQI_FETCH_PTRAID_DATA           (1 << 31)
 977
 978 static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)
 979 {
 980         int rc;
 981         struct bmic_diag_options *diag;
 982
 983         diag = kzalloc(sizeof(*diag), GFP_KERNEL);
 984         if (!diag)
 985                 return -ENOMEM;
 986
 987         rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,
 988                 diag, sizeof(*diag));
 989         if (rc)
 990                 goto out;
 991
 992         diag->options |= cpu_to_le32(PQI_FETCH_PTRAID_DATA);
 993
 994         rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,
 995                 sizeof(*diag));
 996
 997 out:
 998         kfree(diag);
 999
1000         return rc;
1001 }
1002
1003 static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
1004         void *buffer, size_t buffer_length)
1005 {
1006         return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,
1007                 buffer, buffer_length);
1008 }
1009
1010 #pragma pack(1)
1011
1012 struct bmic_host_wellness_driver_version {
1013         u8      start_tag[4];
1014         u8      driver_version_tag[2];
1015         __le16  driver_version_length;
1016         char    driver_version[32];
1017         u8      dont_write_tag[2];
1018         u8      end_tag[2];
1019 };
1020
1021 #pragma pack()
1022
1023 static int pqi_write_driver_version_to_host_wellness(
1024         struct pqi_ctrl_info *ctrl_info)
1025 {
1026         int rc;
1027         struct bmic_host_wellness_driver_version *buffer;
1028         size_t buffer_length;
1029
1030         buffer_length = sizeof(*buffer);
1031
1032         buffer = kmalloc(buffer_length, GFP_KERNEL);
1033         if (!buffer)
1034                 return -ENOMEM;
1035
1036         buffer->start_tag[0] = '<';
1037         buffer->start_tag[1] = 'H';
1038         buffer->start_tag[2] = 'W';
1039         buffer->start_tag[3] = '>';
1040         buffer->driver_version_tag[0] = 'D';
1041         buffer->driver_version_tag[1] = 'V';
1042         put_unaligned_le16(sizeof(buffer->driver_version),
1043                 &buffer->driver_version_length);
1044         strncpy(buffer->driver_version, "Linux " DRIVER_VERSION,
1045                 sizeof(buffer->driver_version) - 1);
1046         buffer->driver_version[sizeof(buffer->driver_version) - 1] = '\0';
1047         buffer->dont_write_tag[0] = 'D';
1048         buffer->dont_write_tag[1] = 'W';
1049         buffer->end_tag[0] = 'Z';
1050         buffer->end_tag[1] = 'Z';
1051
1052         rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
1053
1054         kfree(buffer);
1055
1056         return rc;
1057 }
1058
1059 #pragma pack(1)
1060
1061 struct bmic_host_wellness_time {
1062         u8      start_tag[4];
1063         u8      time_tag[2];
1064         __le16  time_length;
1065         u8      time[8];
1066         u8      dont_write_tag[2];
1067         u8      end_tag[2];
1068 };
1069
1070 #pragma pack()
1071
1072 static int pqi_write_current_time_to_host_wellness(
1073         struct pqi_ctrl_info *ctrl_info)
1074 {
1075         int rc;
1076         struct bmic_host_wellness_time *buffer;
1077         size_t buffer_length;
1078         time64_t local_time;
1079         unsigned int year;
1080         struct tm tm;
1081
1082         buffer_length = sizeof(*buffer);
1083
1084         buffer = kmalloc(buffer_length, GFP_KERNEL);
1085         if (!buffer)
1086                 return -ENOMEM;
1087
1088         buffer->start_tag[0] = '<';
1089         buffer->start_tag[1] = 'H';
1090         buffer->start_tag[2] = 'W';
1091         buffer->start_tag[3] = '>';
1092         buffer->time_tag[0] = 'T';
1093         buffer->time_tag[1] = 'D';
1094         put_unaligned_le16(sizeof(buffer->time),
1095                 &buffer->time_length);
1096
1097         local_time = ktime_get_real_seconds();
1098         time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
1099         year = tm.tm_year + 1900;
1100
1101         buffer->time[0] = bin2bcd(tm.tm_hour);
1102         buffer->time[1] = bin2bcd(tm.tm_min);
1103         buffer->time[2] = bin2bcd(tm.tm_sec);
1104         buffer->time[3] = 0;
1105         buffer->time[4] = bin2bcd(tm.tm_mon + 1);
1106         buffer->time[5] = bin2bcd(tm.tm_mday);
1107         buffer->time[6] = bin2bcd(year / 100);
1108         buffer->time[7] = bin2bcd(year % 100);
1109
1110         buffer->dont_write_tag[0] = 'D';
1111         buffer->dont_write_tag[1] = 'W';
1112         buffer->end_tag[0] = 'Z';
1113         buffer->end_tag[1] = 'Z';
1114
1115         rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
1116
1117         kfree(buffer);
1118
1119         return rc;
1120 }
1121
1122 #define PQI_UPDATE_TIME_WORK_INTERVAL   (24UL * 60 * 60 * HZ)
1123
1124 static void pqi_update_time_worker(struct work_struct *work)
1125 {
1126         int rc;
1127         struct pqi_ctrl_info *ctrl_info;
1128
1129         ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1130                 update_time_work);
1131
1132         rc = pqi_write_current_time_to_host_wellness(ctrl_info);
1133         if (rc)
1134                 dev_warn(&ctrl_info->pci_dev->dev,
1135                         "error updating time on controller\n");
1136
1137         schedule_delayed_work(&ctrl_info->update_time_work,
1138                 PQI_UPDATE_TIME_WORK_INTERVAL);
1139 }
1140
1141 static inline void pqi_schedule_update_time_worker(struct pqi_ctrl_info *ctrl_info)
1142 {
1143         schedule_delayed_work(&ctrl_info->update_time_work, 0);
1144 }
1145
1146 static inline void pqi_cancel_update_time_worker(struct pqi_ctrl_info *ctrl_info)
1147 {
1148         cancel_delayed_work_sync(&ctrl_info->update_time_work);
1149 }
1150
1151 static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void *buffer,
1152         size_t buffer_length)
1153 {
1154         return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer, buffer_length);
1155 }
1156
1157 static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void **buffer)
1158 {
1159         int rc;
1160         size_t lun_list_length;
1161         size_t lun_data_length;
1162         size_t new_lun_list_length;
1163         void *lun_data = NULL;
1164         struct report_lun_header *report_lun_header;
1165
1166         report_lun_header = kmalloc(sizeof(*report_lun_header), GFP_KERNEL);
1167         if (!report_lun_header) {
1168                 rc = -ENOMEM;
1169                 goto out;
1170         }
1171
1172         rc = pqi_report_luns(ctrl_info, cmd, report_lun_header, sizeof(*report_lun_header));
1173         if (rc)
1174                 goto out;
1175
1176         lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
1177
1178 again:
1179         lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
1180
1181         lun_data = kmalloc(lun_data_length, GFP_KERNEL);
1182         if (!lun_data) {
1183                 rc = -ENOMEM;
1184                 goto out;
1185         }
1186
1187         if (lun_list_length == 0) {
1188                 memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
1189                 goto out;
1190         }
1191
1192         rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
1193         if (rc)
1194                 goto out;
1195
1196         new_lun_list_length =
1197                 get_unaligned_be32(&((struct report_lun_header *)lun_data)->list_length);
1198
1199         if (new_lun_list_length > lun_list_length) {
1200                 lun_list_length = new_lun_list_length;
1201                 kfree(lun_data);
1202                 goto again;
1203         }
1204
1205 out:
1206         kfree(report_lun_header);
1207
1208         if (rc) {
1209                 kfree(lun_data);
1210                 lun_data = NULL;
1211         }
1212
1213         *buffer = lun_data;
1214
1215         return rc;
1216 }
1217
1218 static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
1219 {
1220         int rc;
1221         unsigned int i;
1222         u8 rpl_response_format;
1223         u32 num_physicals;
1224         void *rpl_list;
1225         struct report_lun_header *rpl_header;
1226         struct report_phys_lun_8byte_wwid_list *rpl_8byte_wwid_list;
1227         struct report_phys_lun_16byte_wwid_list *rpl_16byte_wwid_list;
1228
1229         rc = pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS, &rpl_list);
1230         if (rc)
1231                 return rc;
1232
1233         if (ctrl_info->rpl_extended_format_4_5_supported) {
1234                 rpl_header = rpl_list;
1235                 rpl_response_format = rpl_header->flags & CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_MASK;
1236                 if (rpl_response_format == CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_4) {
1237                         *buffer = rpl_list;
1238                         return 0;
1239                 } else if (rpl_response_format != CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_2) {
1240                         dev_err(&ctrl_info->pci_dev->dev,
1241                                 "RPL returned unsupported data format %u\n",
1242                                 rpl_response_format);
1243                         return -EINVAL;
1244                 } else {
1245                         dev_warn(&ctrl_info->pci_dev->dev,
1246                                 "RPL returned extended format 2 instead of 4\n");
1247                 }
1248         }
1249
1250         rpl_8byte_wwid_list = rpl_list;
1251         num_physicals = get_unaligned_be32(&rpl_8byte_wwid_list->header.list_length) / sizeof(rpl_8byte_wwid_list->lun_entries[0]);
1252
1253         rpl_16byte_wwid_list = kmalloc(struct_size(rpl_16byte_wwid_list, lun_entries,
1254                                                    num_physicals), GFP_KERNEL);
1255         if (!rpl_16byte_wwid_list)
1256                 return -ENOMEM;
1257
1258         put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid),
1259                 &rpl_16byte_wwid_list->header.list_length);
1260         rpl_16byte_wwid_list->header.flags = rpl_8byte_wwid_list->header.flags;
1261
1262         for (i = 0; i < num_physicals; i++) {
1263                 memcpy(&rpl_16byte_wwid_list->lun_entries[i].lunid, &rpl_8byte_wwid_list->lun_entries[i].lunid, sizeof(rpl_8byte_wwid_list->lun_entries[i].lunid));
1264                 memcpy(&rpl_16byte_wwid_list->lun_entries[i].wwid[0], &rpl_8byte_wwid_list->lun_entries[i].wwid, sizeof(rpl_8byte_wwid_list->lun_entries[i].wwid));
1265                 memset(&rpl_16byte_wwid_list->lun_entries[i].wwid[8], 0, 8);
1266                 rpl_16byte_wwid_list->lun_entries[i].device_type = rpl_8byte_wwid_list->lun_entries[i].device_type;
1267                 rpl_16byte_wwid_list->lun_entries[i].device_flags = rpl_8byte_wwid_list->lun_entries[i].device_flags;
1268                 rpl_16byte_wwid_list->lun_entries[i].lun_count = rpl_8byte_wwid_list->lun_entries[i].lun_count;
1269                 rpl_16byte_wwid_list->lun_entries[i].redundant_paths = rpl_8byte_wwid_list->lun_entries[i].redundant_paths;
1270                 rpl_16byte_wwid_list->lun_entries[i].aio_handle = rpl_8byte_wwid_list->lun_entries[i].aio_handle;
1271         }
1272
1273         kfree(rpl_8byte_wwid_list);
1274         *buffer = rpl_16byte_wwid_list;
1275
1276         return 0;
1277 }
1278
1279 static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
1280 {
1281         return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
1282 }
1283
1284 static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
1285         struct report_phys_lun_16byte_wwid_list **physdev_list,
1286         struct report_log_lun_list **logdev_list)
1287 {
1288         int rc;
1289         size_t logdev_list_length;
1290         size_t logdev_data_length;
1291         struct report_log_lun_list *internal_logdev_list;
1292         struct report_log_lun_list *logdev_data;
1293         struct report_lun_header report_lun_header;
1294
1295         rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
1296         if (rc)
1297                 dev_err(&ctrl_info->pci_dev->dev,
1298                         "report physical LUNs failed\n");
1299
1300         rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
1301         if (rc)
1302                 dev_err(&ctrl_info->pci_dev->dev,
1303                         "report logical LUNs failed\n");
1304
1305         /*
1306          * Tack the controller itself onto the end of the logical device list
1307          * by adding a list entry that is all zeros.
1308          */
1309
1310         logdev_data = *logdev_list;
1311
1312         if (logdev_data) {
1313                 logdev_list_length =
1314                         get_unaligned_be32(&logdev_data->header.list_length);
1315         } else {
1316                 memset(&report_lun_header, 0, sizeof(report_lun_header));
1317                 logdev_data =
1318                         (struct report_log_lun_list *)&report_lun_header;
1319                 logdev_list_length = 0;
1320         }
1321
1322         logdev_data_length = sizeof(struct report_lun_header) +
1323                 logdev_list_length;
1324
1325         internal_logdev_list = kmalloc(logdev_data_length +
1326                 sizeof(struct report_log_lun), GFP_KERNEL);
1327         if (!internal_logdev_list) {
1328                 kfree(*logdev_list);
1329                 *logdev_list = NULL;
1330                 return -ENOMEM;
1331         }
1332
1333         memcpy(internal_logdev_list, logdev_data, logdev_data_length);
1334         memset((u8 *)internal_logdev_list + logdev_data_length, 0,
1335                 sizeof(struct report_log_lun));
1336         put_unaligned_be32(logdev_list_length +
1337                 sizeof(struct report_log_lun),
1338                 &internal_logdev_list->header.list_length);
1339
1340         kfree(*logdev_list);
1341         *logdev_list = internal_logdev_list;
1342
1343         return 0;
1344 }
1345
1346 static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
1347         int bus, int target, int lun)
1348 {
1349         device->bus = bus;
1350         device->target = target;
1351         device->lun = lun;
1352 }
1353
1354 static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
1355 {
1356         u8 *scsi3addr;
1357         u32 lunid;
1358         int bus;
1359         int target;
1360         int lun;
1361
1362         scsi3addr = device->scsi3addr;
1363         lunid = get_unaligned_le32(scsi3addr);
1364
1365         if (pqi_is_hba_lunid(scsi3addr)) {
1366                 /* The specified device is the controller. */
1367                 pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
1368                 device->target_lun_valid = true;
1369                 return;
1370         }
1371
1372         if (pqi_is_logical_device(device)) {
1373                 if (device->is_external_raid_device) {
1374                         bus = PQI_EXTERNAL_RAID_VOLUME_BUS;
1375                         target = (lunid >> 16) & 0x3fff;
1376                         lun = lunid & 0xff;
1377                 } else {
1378                         bus = PQI_RAID_VOLUME_BUS;
1379                         target = 0;
1380                         lun = lunid & 0x3fff;
1381                 }
1382                 pqi_set_bus_target_lun(device, bus, target, lun);
1383                 device->target_lun_valid = true;
1384                 return;
1385         }
1386
1387         /*
1388          * Defer target and LUN assignment for non-controller physical devices
1389          * because the SAS transport layer will make these assignments later.
1390          */
1391         pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
1392 }
1393
1394 static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
1395         struct pqi_scsi_dev *device)
1396 {
1397         int rc;
1398         u8 raid_level;
1399         u8 *buffer;
1400
1401         raid_level = SA_RAID_UNKNOWN;
1402
1403         buffer = kmalloc(64, GFP_KERNEL);
1404         if (buffer) {
1405                 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1406                         VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
1407                 if (rc == 0) {
1408                         raid_level = buffer[8];
1409                         if (raid_level > SA_RAID_MAX)
1410                                 raid_level = SA_RAID_UNKNOWN;
1411                 }
1412                 kfree(buffer);
1413         }
1414
1415         device->raid_level = raid_level;
1416 }
1417
1418 static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
1419         struct pqi_scsi_dev *device, struct raid_map *raid_map)
1420 {
1421         char *err_msg;
1422         u32 raid_map_size;
1423         u32 r5or6_blocks_per_row;
1424
1425         raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1426
1427         if (raid_map_size < offsetof(struct raid_map, disk_data)) {
1428                 err_msg = "RAID map too small";
1429                 goto bad_raid_map;
1430         }
1431
1432         if (device->raid_level == SA_RAID_1) {
1433                 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
1434                         err_msg = "invalid RAID-1 map";
1435                         goto bad_raid_map;
1436                 }
1437         } else if (device->raid_level == SA_RAID_TRIPLE) {
1438                 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
1439                         err_msg = "invalid RAID-1(Triple) map";
1440                         goto bad_raid_map;
1441                 }
1442         } else if ((device->raid_level == SA_RAID_5 ||
1443                 device->raid_level == SA_RAID_6) &&
1444                 get_unaligned_le16(&raid_map->layout_map_count) > 1) {
1445                 /* RAID 50/60 */
1446                 r5or6_blocks_per_row =
1447                         get_unaligned_le16(&raid_map->strip_size) *
1448                         get_unaligned_le16(&raid_map->data_disks_per_row);
1449                 if (r5or6_blocks_per_row == 0) {
1450                         err_msg = "invalid RAID-5 or RAID-6 map";
1451                         goto bad_raid_map;
1452                 }
1453         }
1454
1455         return 0;
1456
1457 bad_raid_map:
1458         dev_warn(&ctrl_info->pci_dev->dev,
1459                 "logical device %08x%08x %s\n",
1460                 *((u32 *)&device->scsi3addr),
1461                 *((u32 *)&device->scsi3addr[4]), err_msg);
1462
1463         return -EINVAL;
1464 }
1465
1466 static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
1467         struct pqi_scsi_dev *device)
1468 {
1469         int rc;
1470         u32 raid_map_size;
1471         struct raid_map *raid_map;
1472
1473         raid_map = kmalloc(sizeof(*raid_map), GFP_KERNEL);
1474         if (!raid_map)
1475                 return -ENOMEM;
1476
1477         rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1478                 device->scsi3addr, raid_map, sizeof(*raid_map), 0, NULL);
1479         if (rc)
1480                 goto error;
1481
1482         raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1483
1484         if (raid_map_size > sizeof(*raid_map)) {
1485
1486                 kfree(raid_map);
1487
1488                 raid_map = kmalloc(raid_map_size, GFP_KERNEL);
1489                 if (!raid_map)
1490                         return -ENOMEM;
1491
1492                 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1493                         device->scsi3addr, raid_map, raid_map_size, 0, NULL);
1494                 if (rc)
1495                         goto error;
1496
1497                 if (get_unaligned_le32(&raid_map->structure_size)
1498                         != raid_map_size) {
1499                         dev_warn(&ctrl_info->pci_dev->dev,
1500                                 "requested %u bytes, received %u bytes\n",
1501                                 raid_map_size,
1502                                 get_unaligned_le32(&raid_map->structure_size));
1503                         rc = -EINVAL;
1504                         goto error;
1505                 }
1506         }
1507
1508         rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
1509         if (rc)
1510                 goto error;
1511
1512         device->raid_map = raid_map;
1513
1514         return 0;
1515
1516 error:
1517         kfree(raid_map);
1518
1519         return rc;
1520 }
1521
1522 static void pqi_set_max_transfer_encrypted(struct pqi_ctrl_info *ctrl_info,
1523         struct pqi_scsi_dev *device)
1524 {
1525         if (!ctrl_info->lv_drive_type_mix_valid) {
1526                 device->max_transfer_encrypted = ~0;
1527                 return;
1528         }
1529
1530         switch (LV_GET_DRIVE_TYPE_MIX(device->scsi3addr)) {
1531         case LV_DRIVE_TYPE_MIX_SAS_HDD_ONLY:
1532         case LV_DRIVE_TYPE_MIX_SATA_HDD_ONLY:
1533         case LV_DRIVE_TYPE_MIX_SAS_OR_SATA_SSD_ONLY:
1534         case LV_DRIVE_TYPE_MIX_SAS_SSD_ONLY:
1535         case LV_DRIVE_TYPE_MIX_SATA_SSD_ONLY:
1536         case LV_DRIVE_TYPE_MIX_SAS_ONLY:
1537         case LV_DRIVE_TYPE_MIX_SATA_ONLY:
1538                 device->max_transfer_encrypted =
1539                         ctrl_info->max_transfer_encrypted_sas_sata;
1540                 break;
1541         case LV_DRIVE_TYPE_MIX_NVME_ONLY:
1542                 device->max_transfer_encrypted =
1543                         ctrl_info->max_transfer_encrypted_nvme;
1544                 break;
1545         case LV_DRIVE_TYPE_MIX_UNKNOWN:
1546         case LV_DRIVE_TYPE_MIX_NO_RESTRICTION:
1547         default:
1548                 device->max_transfer_encrypted =
1549                         min(ctrl_info->max_transfer_encrypted_sas_sata,
1550                                 ctrl_info->max_transfer_encrypted_nvme);
1551                 break;
1552         }
1553 }
1554
1555 static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,
1556         struct pqi_scsi_dev *device)
1557 {
1558         int rc;
1559         u8 *buffer;
1560         u8 bypass_status;
1561
1562         buffer = kmalloc(64, GFP_KERNEL);
1563         if (!buffer)
1564                 return;
1565
1566         rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1567                 VPD_PAGE | CISS_VPD_LV_BYPASS_STATUS, buffer, 64);
1568         if (rc)
1569                 goto out;
1570
1571 #define RAID_BYPASS_STATUS              4
1572 #define RAID_BYPASS_CONFIGURED          0x1
1573 #define RAID_BYPASS_ENABLED             0x2
1574
1575         bypass_status = buffer[RAID_BYPASS_STATUS];
1576         device->raid_bypass_configured =
1577                 (bypass_status & RAID_BYPASS_CONFIGURED) != 0;
1578         if (device->raid_bypass_configured &&
1579                 (bypass_status & RAID_BYPASS_ENABLED) &&
1580                 pqi_get_raid_map(ctrl_info, device) == 0) {
1581                 device->raid_bypass_enabled = true;
1582                 if (get_unaligned_le16(&device->raid_map->flags) &
1583                         RAID_MAP_ENCRYPTION_ENABLED)
1584                         pqi_set_max_transfer_encrypted(ctrl_info, device);
1585         }
1586
1587 out:
1588         kfree(buffer);
1589 }
1590
1591 /*
1592  * Use vendor-specific VPD to determine online/offline status of a volume.
1593  */
1594
1595 static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
1596         struct pqi_scsi_dev *device)
1597 {
1598         int rc;
1599         size_t page_length;
1600         u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
1601         bool volume_offline = true;
1602         u32 volume_flags;
1603         struct ciss_vpd_logical_volume_status *vpd;
1604
1605         vpd = kmalloc(sizeof(*vpd), GFP_KERNEL);
1606         if (!vpd)
1607                 goto no_buffer;
1608
1609         rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1610                 VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
1611         if (rc)
1612                 goto out;
1613
1614         if (vpd->page_code != CISS_VPD_LV_STATUS)
1615                 goto out;
1616
1617         page_length = offsetof(struct ciss_vpd_logical_volume_status,
1618                 volume_status) + vpd->page_length;
1619         if (page_length < sizeof(*vpd))
1620                 goto out;
1621
1622         volume_status = vpd->volume_status;
1623         volume_flags = get_unaligned_be32(&vpd->flags);
1624         volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
1625
1626 out:
1627         kfree(vpd);
1628 no_buffer:
1629         device->volume_status = volume_status;
1630         device->volume_offline = volume_offline;
1631 }
1632
1633 #define PQI_DEVICE_NCQ_PRIO_SUPPORTED   0x01
1634 #define PQI_DEVICE_PHY_MAP_SUPPORTED    0x10
1635 #define PQI_DEVICE_ERASE_IN_PROGRESS    0x10
1636
1637 static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,
1638         struct pqi_scsi_dev *device,
1639         struct bmic_identify_physical_device *id_phys)
1640 {
1641         int rc;
1642
1643         memset(id_phys, 0, sizeof(*id_phys));
1644
1645         rc = pqi_identify_physical_device(ctrl_info, device,
1646                 id_phys, sizeof(*id_phys));
1647         if (rc) {
1648                 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
1649                 return rc;
1650         }
1651
1652         scsi_sanitize_inquiry_string(&id_phys->model[0], 8);
1653         scsi_sanitize_inquiry_string(&id_phys->model[8], 16);
1654
1655         memcpy(device->vendor, &id_phys->model[0], sizeof(device->vendor));
1656         memcpy(device->model, &id_phys->model[8], sizeof(device->model));
1657
1658         device->box_index = id_phys->box_index;
1659         device->phys_box_on_bus = id_phys->phys_box_on_bus;
1660         device->phy_connected_dev_type = id_phys->phy_connected_dev_type[0];
1661         device->queue_depth =
1662                 get_unaligned_le16(&id_phys->current_queue_depth_limit);
1663         device->active_path_index = id_phys->active_path_number;
1664         device->path_map = id_phys->redundant_path_present_map;
1665         memcpy(&device->box,
1666                 &id_phys->alternate_paths_phys_box_on_port,
1667                 sizeof(device->box));
1668         memcpy(&device->phys_connector,
1669                 &id_phys->alternate_paths_phys_connector,
1670                 sizeof(device->phys_connector));
1671         device->bay = id_phys->phys_bay_in_box;
1672         device->lun_count = id_phys->multi_lun_device_lun_count;
1673         if ((id_phys->even_more_flags & PQI_DEVICE_PHY_MAP_SUPPORTED) &&
1674                 id_phys->phy_count)
1675                 device->phy_id =
1676                         id_phys->phy_to_phy_map[device->active_path_index];
1677         else
1678                 device->phy_id = 0xFF;
1679
1680         device->ncq_prio_support =
1681                 ((get_unaligned_le32(&id_phys->misc_drive_flags) >> 16) &
1682                 PQI_DEVICE_NCQ_PRIO_SUPPORTED);
1683
1684         device->erase_in_progress = !!(get_unaligned_le16(&id_phys->extra_physical_drive_flags) & PQI_DEVICE_ERASE_IN_PROGRESS);
1685
1686         return 0;
1687 }
1688
1689 static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,
1690         struct pqi_scsi_dev *device)
1691 {
1692         int rc;
1693         u8 *buffer;
1694
1695         buffer = kmalloc(64, GFP_KERNEL);
1696         if (!buffer)
1697                 return -ENOMEM;
1698
1699         /* Send an inquiry to the device to see what it is. */
1700         rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
1701         if (rc)
1702                 goto out;
1703
1704         scsi_sanitize_inquiry_string(&buffer[8], 8);
1705         scsi_sanitize_inquiry_string(&buffer[16], 16);
1706
1707         device->devtype = buffer[0] & 0x1f;
1708         memcpy(device->vendor, &buffer[8], sizeof(device->vendor));
1709         memcpy(device->model, &buffer[16], sizeof(device->model));
1710
1711         if (device->devtype == TYPE_DISK) {
1712                 if (device->is_external_raid_device) {
1713                         device->raid_level = SA_RAID_UNKNOWN;
1714                         device->volume_status = CISS_LV_OK;
1715                         device->volume_offline = false;
1716                 } else {
1717                         pqi_get_raid_level(ctrl_info, device);
1718                         pqi_get_raid_bypass_status(ctrl_info, device);
1719                         pqi_get_volume_status(ctrl_info, device);
1720                 }
1721         }
1722
1723 out:
1724         kfree(buffer);
1725
1726         return rc;
1727 }
1728
1729 /*
1730  * Prevent adding drive to OS for some corner cases such as a drive
1731  * undergoing a sanitize (erase) operation. Some OSes will continue to poll
1732  * the drive until the sanitize completes, which can take hours,
1733  * resulting in long bootup delays. Commands such as TUR, READ_CAP
1734  * are allowed, but READ/WRITE cause check condition. So the OS
1735  * cannot check/read the partition table.
1736  * Note: devices that have completed sanitize must be re-enabled
1737  *       using the management utility.
1738  */
1739 static inline bool pqi_keep_device_offline(struct pqi_scsi_dev *device)
1740 {
1741         return device->erase_in_progress;
1742 }
1743
1744 static int pqi_get_device_info_phys_logical(struct pqi_ctrl_info *ctrl_info,
1745         struct pqi_scsi_dev *device,
1746         struct bmic_identify_physical_device *id_phys)
1747 {
1748         int rc;
1749
1750         if (device->is_expander_smp_device)
1751                 return 0;
1752
1753         if (pqi_is_logical_device(device))
1754                 rc = pqi_get_logical_device_info(ctrl_info, device);
1755         else
1756                 rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);
1757
1758         return rc;
1759 }
1760
1761 static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
1762         struct pqi_scsi_dev *device,
1763         struct bmic_identify_physical_device *id_phys)
1764 {
1765         int rc;
1766
1767         rc = pqi_get_device_info_phys_logical(ctrl_info, device, id_phys);
1768
1769         if (rc == 0 && device->lun_count == 0)
1770                 device->lun_count = 1;
1771
1772         return rc;
1773 }
1774
1775 static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
1776         struct pqi_scsi_dev *device)
1777 {
1778         char *status;
1779         static const char unknown_state_str[] =
1780                 "Volume is in an unknown state (%u)";
1781         char unknown_state_buffer[sizeof(unknown_state_str) + 10];
1782
1783         switch (device->volume_status) {
1784         case CISS_LV_OK:
1785                 status = "Volume online";
1786                 break;
1787         case CISS_LV_FAILED:
1788                 status = "Volume failed";
1789                 break;
1790         case CISS_LV_NOT_CONFIGURED:
1791                 status = "Volume not configured";
1792                 break;
1793         case CISS_LV_DEGRADED:
1794                 status = "Volume degraded";
1795                 break;
1796         case CISS_LV_READY_FOR_RECOVERY:
1797                 status = "Volume ready for recovery operation";
1798                 break;
1799         case CISS_LV_UNDERGOING_RECOVERY:
1800                 status = "Volume undergoing recovery";
1801                 break;
1802         case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
1803                 status = "Wrong physical drive was replaced";
1804                 break;
1805         case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
1806                 status = "A physical drive not properly connected";
1807                 break;
1808         case CISS_LV_HARDWARE_OVERHEATING:
1809                 status = "Hardware is overheating";
1810                 break;
1811         case CISS_LV_HARDWARE_HAS_OVERHEATED:
1812                 status = "Hardware has overheated";
1813                 break;
1814         case CISS_LV_UNDERGOING_EXPANSION:
1815                 status = "Volume undergoing expansion";
1816                 break;
1817         case CISS_LV_NOT_AVAILABLE:
1818                 status = "Volume waiting for transforming volume";
1819                 break;
1820         case CISS_LV_QUEUED_FOR_EXPANSION:
1821                 status = "Volume queued for expansion";
1822                 break;
1823         case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
1824                 status = "Volume disabled due to SCSI ID conflict";
1825                 break;
1826         case CISS_LV_EJECTED:
1827                 status = "Volume has been ejected";
1828                 break;
1829         case CISS_LV_UNDERGOING_ERASE:
1830                 status = "Volume undergoing background erase";
1831                 break;
1832         case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
1833                 status = "Volume ready for predictive spare rebuild";
1834                 break;
1835         case CISS_LV_UNDERGOING_RPI:
1836                 status = "Volume undergoing rapid parity initialization";
1837                 break;
1838         case CISS_LV_PENDING_RPI:
1839                 status = "Volume queued for rapid parity initialization";
1840                 break;
1841         case CISS_LV_ENCRYPTED_NO_KEY:
1842                 status = "Encrypted volume inaccessible - key not present";
1843                 break;
1844         case CISS_LV_UNDERGOING_ENCRYPTION:
1845                 status = "Volume undergoing encryption process";
1846                 break;
1847         case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
1848                 status = "Volume undergoing encryption re-keying process";
1849                 break;
1850         case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
1851                 status = "Volume encrypted but encryption is disabled";
1852                 break;
1853         case CISS_LV_PENDING_ENCRYPTION:
1854                 status = "Volume pending migration to encrypted state";
1855                 break;
1856         case CISS_LV_PENDING_ENCRYPTION_REKEYING:
1857                 status = "Volume pending encryption rekeying";
1858                 break;
1859         case CISS_LV_NOT_SUPPORTED:
1860                 status = "Volume not supported on this controller";
1861                 break;
1862         case CISS_LV_STATUS_UNAVAILABLE:
1863                 status = "Volume status not available";
1864                 break;
1865         default:
1866                 snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
1867                         unknown_state_str, device->volume_status);
1868                 status = unknown_state_buffer;
1869                 break;
1870         }
1871
1872         dev_info(&ctrl_info->pci_dev->dev,
1873                 "scsi %d:%d:%d:%d %s\n",
1874                 ctrl_info->scsi_host->host_no,
1875                 device->bus, device->target, device->lun, status);
1876 }
1877
1878 static void pqi_rescan_worker(struct work_struct *work)
1879 {
1880         struct pqi_ctrl_info *ctrl_info;
1881
1882         ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1883                 rescan_work);
1884
1885         pqi_scan_scsi_devices(ctrl_info);
1886 }
1887
1888 static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
1889         struct pqi_scsi_dev *device)
1890 {
1891         int rc;
1892
1893         if (pqi_is_logical_device(device))
1894                 rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
1895                         device->target, device->lun);
1896         else
1897                 rc = pqi_add_sas_device(ctrl_info->sas_host, device);
1898
1899         return rc;
1900 }
1901
1902 #define PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS      (20 * 1000)
1903
1904 static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)
1905 {
1906         int rc;
1907         int lun;
1908
1909         for (lun = 0; lun < device->lun_count; lun++) {
1910                 rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun,
1911                         PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS);
1912                 if (rc)
1913                         dev_err(&ctrl_info->pci_dev->dev,
1914                                 "scsi %d:%d:%d:%d removing device with %d outstanding command(s)\n",
1915                                 ctrl_info->scsi_host->host_no, device->bus,
1916                                 device->target, lun,
1917                                 atomic_read(&device->scsi_cmds_outstanding[lun]));
1918         }
1919
1920         if (pqi_is_logical_device(device))
1921                 scsi_remove_device(device->sdev);
1922         else
1923                 pqi_remove_sas_device(device);
1924
1925         pqi_device_remove_start(device);
1926 }
1927
1928 /* Assumes the SCSI device list lock is held. */
1929
1930 static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
1931         int bus, int target, int lun)
1932 {
1933         struct pqi_scsi_dev *device;
1934
1935         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
1936                 if (device->bus == bus && device->target == target && device->lun == lun)
1937                         return device;
1938
1939         return NULL;
1940 }
1941
1942 static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1, struct pqi_scsi_dev *dev2)
1943 {
1944         if (dev1->is_physical_device != dev2->is_physical_device)
1945                 return false;
1946
1947         if (dev1->is_physical_device)
1948                 return memcmp(dev1->wwid, dev2->wwid, sizeof(dev1->wwid)) == 0;
1949
1950         return memcmp(dev1->volume_id, dev2->volume_id, sizeof(dev1->volume_id)) == 0;
1951 }
1952
1953 enum pqi_find_result {
1954         DEVICE_NOT_FOUND,
1955         DEVICE_CHANGED,
1956         DEVICE_SAME,
1957 };
1958
1959 static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
1960         struct pqi_scsi_dev *device_to_find, struct pqi_scsi_dev **matching_device)
1961 {
1962         struct pqi_scsi_dev *device;
1963
1964         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
1965                 if (pqi_scsi3addr_equal(device_to_find->scsi3addr, device->scsi3addr)) {
1966                         *matching_device = device;
1967                         if (pqi_device_equal(device_to_find, device)) {
1968                                 if (device_to_find->volume_offline)
1969                                         return DEVICE_CHANGED;
1970                                 return DEVICE_SAME;
1971                         }
1972                         return DEVICE_CHANGED;
1973                 }
1974         }
1975
1976         return DEVICE_NOT_FOUND;
1977 }
1978
1979 static inline const char *pqi_device_type(struct pqi_scsi_dev *device)
1980 {
1981         if (device->is_expander_smp_device)
1982                 return "Enclosure SMP    ";
1983
1984         return scsi_device_type(device->devtype);
1985 }
1986
1987 #define PQI_DEV_INFO_BUFFER_LENGTH      128
1988
1989 static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
1990         char *action, struct pqi_scsi_dev *device)
1991 {
1992         ssize_t count;
1993         char buffer[PQI_DEV_INFO_BUFFER_LENGTH];
1994
1995         count = scnprintf(buffer, PQI_DEV_INFO_BUFFER_LENGTH,
1996                 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus);
1997
1998         if (device->target_lun_valid)
1999                 count += scnprintf(buffer + count,
2000                         PQI_DEV_INFO_BUFFER_LENGTH - count,
2001                         "%d:%d",
2002                         device->target,
2003                         device->lun);
2004         else
2005                 count += scnprintf(buffer + count,
2006                         PQI_DEV_INFO_BUFFER_LENGTH - count,
2007                         "-:-");
2008
2009         if (pqi_is_logical_device(device))
2010                 count += scnprintf(buffer + count,
2011                         PQI_DEV_INFO_BUFFER_LENGTH - count,
2012                         " %08x%08x",
2013                         *((u32 *)&device->scsi3addr),
2014                         *((u32 *)&device->scsi3addr[4]));
2015         else
2016                 count += scnprintf(buffer + count,
2017                         PQI_DEV_INFO_BUFFER_LENGTH - count,
2018                         " %016llx%016llx",
2019                         get_unaligned_be64(&device->wwid[0]),
2020                         get_unaligned_be64(&device->wwid[8]));
2021
2022         count += scnprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count,
2023                 " %s %.8s %.16s ",
2024                 pqi_device_type(device),
2025                 device->vendor,
2026                 device->model);
2027
2028         if (pqi_is_logical_device(device)) {
2029                 if (device->devtype == TYPE_DISK)
2030                         count += scnprintf(buffer + count,
2031                                 PQI_DEV_INFO_BUFFER_LENGTH - count,
2032                                 "SSDSmartPathCap%c En%c %-12s",
2033                                 device->raid_bypass_configured ? '+' : '-',
2034                                 device->raid_bypass_enabled ? '+' : '-',
2035                                 pqi_raid_level_to_string(device->raid_level));
2036         } else {
2037                 count += scnprintf(buffer + count,
2038                         PQI_DEV_INFO_BUFFER_LENGTH - count,
2039                         "AIO%c", device->aio_enabled ? '+' : '-');
2040                 if (device->devtype == TYPE_DISK ||
2041                         device->devtype == TYPE_ZBC)
2042                         count += scnprintf(buffer + count,
2043                                 PQI_DEV_INFO_BUFFER_LENGTH - count,
2044                                 " qd=%-6d", device->queue_depth);
2045         }
2046
2047         dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);
2048 }
2049
2050 static bool pqi_raid_maps_equal(struct raid_map *raid_map1, struct raid_map *raid_map2)
2051 {
2052         u32 raid_map1_size;
2053         u32 raid_map2_size;
2054
2055         if (raid_map1 == NULL || raid_map2 == NULL)
2056                 return raid_map1 == raid_map2;
2057
2058         raid_map1_size = get_unaligned_le32(&raid_map1->structure_size);
2059         raid_map2_size = get_unaligned_le32(&raid_map2->structure_size);
2060
2061         if (raid_map1_size != raid_map2_size)
2062                 return false;
2063
2064         return memcmp(raid_map1, raid_map2, raid_map1_size) == 0;
2065 }
2066
2067 /* Assumes the SCSI device list lock is held. */
2068
2069 static void pqi_scsi_update_device(struct pqi_ctrl_info *ctrl_info,
2070         struct pqi_scsi_dev *existing_device, struct pqi_scsi_dev *new_device)
2071 {
2072         existing_device->device_type = new_device->device_type;
2073         existing_device->bus = new_device->bus;
2074         if (new_device->target_lun_valid) {
2075                 existing_device->target = new_device->target;
2076                 existing_device->lun = new_device->lun;
2077                 existing_device->target_lun_valid = true;
2078         }
2079
2080         /* By definition, the scsi3addr and wwid fields are already the same. */
2081
2082         existing_device->is_physical_device = new_device->is_physical_device;
2083         memcpy(existing_device->vendor, new_device->vendor, sizeof(existing_device->vendor));
2084         memcpy(existing_device->model, new_device->model, sizeof(existing_device->model));
2085         existing_device->sas_address = new_device->sas_address;
2086         existing_device->queue_depth = new_device->queue_depth;
2087         existing_device->device_offline = false;
2088         existing_device->lun_count = new_device->lun_count;
2089
2090         if (pqi_is_logical_device(existing_device)) {
2091                 existing_device->is_external_raid_device = new_device->is_external_raid_device;
2092
2093                 if (existing_device->devtype == TYPE_DISK) {
2094                         existing_device->raid_level = new_device->raid_level;
2095                         existing_device->volume_status = new_device->volume_status;
2096                         memset(existing_device->next_bypass_group, 0, sizeof(existing_device->next_bypass_group));
2097                         if (!pqi_raid_maps_equal(existing_device->raid_map, new_device->raid_map)) {
2098                                 kfree(existing_device->raid_map);
2099                                 existing_device->raid_map = new_device->raid_map;
2100                                 /* To prevent this from being freed later. */
2101                                 new_device->raid_map = NULL;
2102                         }
2103                         existing_device->raid_bypass_configured = new_device->raid_bypass_configured;
2104                         existing_device->raid_bypass_enabled = new_device->raid_bypass_enabled;
2105                 }
2106         } else {
2107                 existing_device->aio_enabled = new_device->aio_enabled;
2108                 existing_device->aio_handle = new_device->aio_handle;
2109                 existing_device->is_expander_smp_device = new_device->is_expander_smp_device;
2110                 existing_device->active_path_index = new_device->active_path_index;
2111                 existing_device->phy_id = new_device->phy_id;
2112                 existing_device->path_map = new_device->path_map;
2113                 existing_device->bay = new_device->bay;
2114                 existing_device->box_index = new_device->box_index;
2115                 existing_device->phys_box_on_bus = new_device->phys_box_on_bus;
2116                 existing_device->phy_connected_dev_type = new_device->phy_connected_dev_type;
2117                 memcpy(existing_device->box, new_device->box, sizeof(existing_device->box));
2118                 memcpy(existing_device->phys_connector, new_device->phys_connector, sizeof(existing_device->phys_connector));
2119         }
2120 }
2121
2122 static inline void pqi_free_device(struct pqi_scsi_dev *device)
2123 {
2124         if (device) {
2125                 kfree(device->raid_map);
2126                 kfree(device);
2127         }
2128 }
2129
2130 /*
2131  * Called when exposing a new device to the OS fails in order to re-adjust
2132  * our internal SCSI device list to match the SCSI ML's view.
2133  */
2134
2135 static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
2136         struct pqi_scsi_dev *device)
2137 {
2138         unsigned long flags;
2139
2140         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2141         list_del(&device->scsi_device_list_entry);
2142         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2143
2144         /* Allow the device structure to be freed later. */
2145         device->keep_device = false;
2146 }
2147
2148 static inline bool pqi_is_device_added(struct pqi_scsi_dev *device)
2149 {
2150         if (device->is_expander_smp_device)
2151                 return device->sas_port != NULL;
2152
2153         return device->sdev != NULL;
2154 }
2155
2156 static inline void pqi_init_device_tmf_work(struct pqi_scsi_dev *device)
2157 {
2158         unsigned int lun;
2159         struct pqi_tmf_work *tmf_work;
2160
2161         for (lun = 0, tmf_work = device->tmf_work; lun < PQI_MAX_LUNS_PER_DEVICE; lun++, tmf_work++)
2162                 INIT_WORK(&tmf_work->work_struct, pqi_tmf_worker);
2163 }
2164
2165 static inline bool pqi_volume_rescan_needed(struct pqi_scsi_dev *device)
2166 {
2167         if (pqi_device_in_remove(device))
2168                 return false;
2169
2170         if (device->sdev == NULL)
2171                 return false;
2172
2173         if (!scsi_device_online(device->sdev))
2174                 return false;
2175
2176         return device->rescan;
2177 }
2178
2179 static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
2180         struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
2181 {
2182         int rc;
2183         unsigned int i;
2184         unsigned long flags;
2185         enum pqi_find_result find_result;
2186         struct pqi_scsi_dev *device;
2187         struct pqi_scsi_dev *next;
2188         struct pqi_scsi_dev *matching_device;
2189         LIST_HEAD(add_list);
2190         LIST_HEAD(delete_list);
2191
2192         /*
2193          * The idea here is to do as little work as possible while holding the
2194          * spinlock.  That's why we go to great pains to defer anything other
2195          * than updating the internal device list until after we release the
2196          * spinlock.
2197          */
2198
2199         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2200
2201         /* Assume that all devices in the existing list have gone away. */
2202         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
2203                 device->device_gone = true;
2204
2205         for (i = 0; i < num_new_devices; i++) {
2206                 device = new_device_list[i];
2207
2208                 find_result = pqi_scsi_find_entry(ctrl_info, device,
2209                         &matching_device);
2210
2211                 switch (find_result) {
2212                 case DEVICE_SAME:
2213                         /*
2214                          * The newly found device is already in the existing
2215                          * device list.
2216                          */
2217                         device->new_device = false;
2218                         matching_device->device_gone = false;
2219                         pqi_scsi_update_device(ctrl_info, matching_device, device);
2220                         break;
2221                 case DEVICE_NOT_FOUND:
2222                         /*
2223                          * The newly found device is NOT in the existing device
2224                          * list.
2225                          */
2226                         device->new_device = true;
2227                         break;
2228                 case DEVICE_CHANGED:
2229                         /*
2230                          * The original device has gone away and we need to add
2231                          * the new device.
2232                          */
2233                         device->new_device = true;
2234                         break;
2235                 }
2236         }
2237
2238         /* Process all devices that have gone away. */
2239         list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
2240                 scsi_device_list_entry) {
2241                 if (device->device_gone) {
2242                         list_del(&device->scsi_device_list_entry);
2243                         list_add_tail(&device->delete_list_entry, &delete_list);
2244                 }
2245         }
2246
2247         /* Process all new devices. */
2248         for (i = 0; i < num_new_devices; i++) {
2249                 device = new_device_list[i];
2250                 if (!device->new_device)
2251                         continue;
2252                 if (device->volume_offline)
2253                         continue;
2254                 list_add_tail(&device->scsi_device_list_entry,
2255                         &ctrl_info->scsi_device_list);
2256                 list_add_tail(&device->add_list_entry, &add_list);
2257                 /* To prevent this device structure from being freed later. */
2258                 device->keep_device = true;
2259                 pqi_init_device_tmf_work(device);
2260         }
2261
2262         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2263
2264         /*
2265          * If OFA is in progress and there are devices that need to be deleted,
2266          * allow any pending reset operations to continue and unblock any SCSI
2267          * requests before removal.
2268          */
2269         if (pqi_ofa_in_progress(ctrl_info)) {
2270                 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry)
2271                         if (pqi_is_device_added(device))
2272                                 pqi_device_remove_start(device);
2273                 pqi_ctrl_unblock_device_reset(ctrl_info);
2274                 pqi_scsi_unblock_requests(ctrl_info);
2275         }
2276
2277         /* Remove all devices that have gone away. */
2278         list_for_each_entry_safe(device, next, &delete_list, delete_list_entry) {
2279                 if (device->volume_offline) {
2280                         pqi_dev_info(ctrl_info, "offline", device);
2281                         pqi_show_volume_status(ctrl_info, device);
2282                 } else {
2283                         pqi_dev_info(ctrl_info, "removed", device);
2284                 }
2285                 if (pqi_is_device_added(device))
2286                         pqi_remove_device(ctrl_info, device);
2287                 list_del(&device->delete_list_entry);
2288                 pqi_free_device(device);
2289         }
2290
2291         /*
2292          * Notify the SML of any existing device changes such as;
2293          * queue depth, device size.
2294          */
2295         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
2296                 if (device->sdev && device->queue_depth != device->advertised_queue_depth) {
2297                         device->advertised_queue_depth = device->queue_depth;
2298                         scsi_change_queue_depth(device->sdev, device->advertised_queue_depth);
2299                         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2300                         if (pqi_volume_rescan_needed(device)) {
2301                                 device->rescan = false;
2302                                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2303                                 scsi_rescan_device(device->sdev);
2304                         } else {
2305                                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2306                         }
2307                 }
2308         }
2309
2310         /* Expose any new devices. */
2311         list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
2312                 if (!pqi_is_device_added(device)) {
2313                         rc = pqi_add_device(ctrl_info, device);
2314                         if (rc == 0) {
2315                                 pqi_dev_info(ctrl_info, "added", device);
2316                         } else {
2317                                 dev_warn(&ctrl_info->pci_dev->dev,
2318                                         "scsi %d:%d:%d:%d addition failed, device not added\n",
2319                                         ctrl_info->scsi_host->host_no,
2320                                         device->bus, device->target,
2321                                         device->lun);
2322                                 pqi_fixup_botched_add(ctrl_info, device);
2323                         }
2324                 }
2325         }
2326
2327 }
2328
2329 static inline bool pqi_is_supported_device(struct pqi_scsi_dev *device)
2330 {
2331         /*
2332          * Only support the HBA controller itself as a RAID
2333          * controller.  If it's a RAID controller other than
2334          * the HBA itself (an external RAID controller, for
2335          * example), we don't support it.
2336          */
2337         if (device->device_type == SA_DEVICE_TYPE_CONTROLLER &&
2338                 !pqi_is_hba_lunid(device->scsi3addr))
2339                         return false;
2340
2341         return true;
2342 }
2343
2344 static inline bool pqi_skip_device(u8 *scsi3addr)
2345 {
2346         /* Ignore all masked devices. */
2347         if (MASKED_DEVICE(scsi3addr))
2348                 return true;
2349
2350         return false;
2351 }
2352
2353 static inline void pqi_mask_device(u8 *scsi3addr)
2354 {
2355         scsi3addr[3] |= 0xc0;
2356 }
2357
2358 static inline bool pqi_is_multipath_device(struct pqi_scsi_dev *device)
2359 {
2360         if (pqi_is_logical_device(device))
2361                 return false;
2362
2363         return (device->path_map & (device->path_map - 1)) != 0;
2364 }
2365
2366 static inline bool pqi_expose_device(struct pqi_scsi_dev *device)
2367 {
2368         return !device->is_physical_device || !pqi_skip_device(device->scsi3addr);
2369 }
2370
2371 static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2372 {
2373         int i;
2374         int rc;
2375         LIST_HEAD(new_device_list_head);
2376         struct report_phys_lun_16byte_wwid_list *physdev_list = NULL;
2377         struct report_log_lun_list *logdev_list = NULL;
2378         struct report_phys_lun_16byte_wwid *phys_lun;
2379         struct report_log_lun *log_lun;
2380         struct bmic_identify_physical_device *id_phys = NULL;
2381         u32 num_physicals;
2382         u32 num_logicals;
2383         struct pqi_scsi_dev **new_device_list = NULL;
2384         struct pqi_scsi_dev *device;
2385         struct pqi_scsi_dev *next;
2386         unsigned int num_new_devices;
2387         unsigned int num_valid_devices;
2388         bool is_physical_device;
2389         u8 *scsi3addr;
2390         unsigned int physical_index;
2391         unsigned int logical_index;
2392         static char *out_of_memory_msg =
2393                 "failed to allocate memory, device discovery stopped";
2394
2395         rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
2396         if (rc)
2397                 goto out;
2398
2399         if (physdev_list)
2400                 num_physicals =
2401                         get_unaligned_be32(&physdev_list->header.list_length)
2402                                 / sizeof(physdev_list->lun_entries[0]);
2403         else
2404                 num_physicals = 0;
2405
2406         if (logdev_list)
2407                 num_logicals =
2408                         get_unaligned_be32(&logdev_list->header.list_length)
2409                                 / sizeof(logdev_list->lun_entries[0]);
2410         else
2411                 num_logicals = 0;
2412
2413         if (num_physicals) {
2414                 /*
2415                  * We need this buffer for calls to pqi_get_physical_disk_info()
2416                  * below.  We allocate it here instead of inside
2417                  * pqi_get_physical_disk_info() because it's a fairly large
2418                  * buffer.
2419                  */
2420                 id_phys = kmalloc(sizeof(*id_phys), GFP_KERNEL);
2421                 if (!id_phys) {
2422                         dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2423                                 out_of_memory_msg);
2424                         rc = -ENOMEM;
2425                         goto out;
2426                 }
2427
2428                 if (pqi_hide_vsep) {
2429                         for (i = num_physicals - 1; i >= 0; i--) {
2430                                 phys_lun = &physdev_list->lun_entries[i];
2431                                 if (CISS_GET_DRIVE_NUMBER(phys_lun->lunid) == PQI_VSEP_CISS_BTL) {
2432                                         pqi_mask_device(phys_lun->lunid);
2433                                         break;
2434                                 }
2435                         }
2436                 }
2437         }
2438
2439         if (num_logicals &&
2440                 (logdev_list->header.flags & CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX))
2441                 ctrl_info->lv_drive_type_mix_valid = true;
2442
2443         num_new_devices = num_physicals + num_logicals;
2444
2445         new_device_list = kmalloc_array(num_new_devices,
2446                                         sizeof(*new_device_list),
2447                                         GFP_KERNEL);
2448         if (!new_device_list) {
2449                 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
2450                 rc = -ENOMEM;
2451                 goto out;
2452         }
2453
2454         for (i = 0; i < num_new_devices; i++) {
2455                 device = kzalloc(sizeof(*device), GFP_KERNEL);
2456                 if (!device) {
2457                         dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2458                                 out_of_memory_msg);
2459                         rc = -ENOMEM;
2460                         goto out;
2461                 }
2462                 list_add_tail(&device->new_device_list_entry,
2463                         &new_device_list_head);
2464         }
2465
2466         device = NULL;
2467         num_valid_devices = 0;
2468         physical_index = 0;
2469         logical_index = 0;
2470
2471         for (i = 0; i < num_new_devices; i++) {
2472
2473                 if ((!pqi_expose_ld_first && i < num_physicals) ||
2474                         (pqi_expose_ld_first && i >= num_logicals)) {
2475                         is_physical_device = true;
2476                         phys_lun = &physdev_list->lun_entries[physical_index++];
2477                         log_lun = NULL;
2478                         scsi3addr = phys_lun->lunid;
2479                 } else {
2480                         is_physical_device = false;
2481                         phys_lun = NULL;
2482                         log_lun = &logdev_list->lun_entries[logical_index++];
2483                         scsi3addr = log_lun->lunid;
2484                 }
2485
2486                 if (is_physical_device && pqi_skip_device(scsi3addr))
2487                         continue;
2488
2489                 if (device)
2490                         device = list_next_entry(device, new_device_list_entry);
2491                 else
2492                         device = list_first_entry(&new_device_list_head,
2493                                 struct pqi_scsi_dev, new_device_list_entry);
2494
2495                 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
2496                 device->is_physical_device = is_physical_device;
2497                 if (is_physical_device) {
2498                         device->device_type = phys_lun->device_type;
2499                         if (device->device_type == SA_DEVICE_TYPE_EXPANDER_SMP)
2500                                 device->is_expander_smp_device = true;
2501                 } else {
2502                         device->is_external_raid_device =
2503                                 pqi_is_external_raid_addr(scsi3addr);
2504                 }
2505
2506                 if (!pqi_is_supported_device(device))
2507                         continue;
2508
2509                 /* Gather information about the device. */
2510                 rc = pqi_get_device_info(ctrl_info, device, id_phys);
2511                 if (rc == -ENOMEM) {
2512                         dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2513                                 out_of_memory_msg);
2514                         goto out;
2515                 }
2516                 if (rc) {
2517                         if (device->is_physical_device)
2518                                 dev_warn(&ctrl_info->pci_dev->dev,
2519                                         "obtaining device info failed, skipping physical device %016llx%016llx\n",
2520                                         get_unaligned_be64(&phys_lun->wwid[0]),
2521                                         get_unaligned_be64(&phys_lun->wwid[8]));
2522                         else
2523                                 dev_warn(&ctrl_info->pci_dev->dev,
2524                                         "obtaining device info failed, skipping logical device %08x%08x\n",
2525                                         *((u32 *)&device->scsi3addr),
2526                                         *((u32 *)&device->scsi3addr[4]));
2527                         rc = 0;
2528                         continue;
2529                 }
2530
2531                 /* Do not present disks that the OS cannot fully probe. */
2532                 if (pqi_keep_device_offline(device))
2533                         continue;
2534
2535                 pqi_assign_bus_target_lun(device);
2536
2537                 if (device->is_physical_device) {
2538                         memcpy(device->wwid, phys_lun->wwid, sizeof(device->wwid));
2539                         if ((phys_lun->device_flags &
2540                                 CISS_REPORT_PHYS_DEV_FLAG_AIO_ENABLED) &&
2541                                 phys_lun->aio_handle) {
2542                                         device->aio_enabled = true;
2543                                         device->aio_handle =
2544                                                 phys_lun->aio_handle;
2545                         }
2546                 } else {
2547                         memcpy(device->volume_id, log_lun->volume_id,
2548                                 sizeof(device->volume_id));
2549                 }
2550
2551                 device->sas_address = get_unaligned_be64(&device->wwid[0]);
2552
2553                 new_device_list[num_valid_devices++] = device;
2554         }
2555
2556         pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
2557
2558 out:
2559         list_for_each_entry_safe(device, next, &new_device_list_head,
2560                 new_device_list_entry) {
2561                 if (device->keep_device)
2562                         continue;
2563                 list_del(&device->new_device_list_entry);
2564                 pqi_free_device(device);
2565         }
2566
2567         kfree(new_device_list);
2568         kfree(physdev_list);
2569         kfree(logdev_list);
2570         kfree(id_phys);
2571
2572         return rc;
2573 }
2574
2575 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2576 {
2577         int rc;
2578         int mutex_acquired;
2579
2580         if (pqi_ctrl_offline(ctrl_info))
2581                 return -ENXIO;
2582
2583         mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);
2584
2585         if (!mutex_acquired) {
2586                 if (pqi_ctrl_scan_blocked(ctrl_info))
2587                         return -EBUSY;
2588                 pqi_schedule_rescan_worker_delayed(ctrl_info);
2589                 return -EINPROGRESS;
2590         }
2591
2592         rc = pqi_update_scsi_devices(ctrl_info);
2593         if (rc && !pqi_ctrl_scan_blocked(ctrl_info))
2594                 pqi_schedule_rescan_worker_delayed(ctrl_info);
2595
2596         mutex_unlock(&ctrl_info->scan_mutex);
2597
2598         return rc;
2599 }
2600
2601 static void pqi_scan_start(struct Scsi_Host *shost)
2602 {
2603         struct pqi_ctrl_info *ctrl_info;
2604
2605         ctrl_info = shost_to_hba(shost);
2606
2607         pqi_scan_scsi_devices(ctrl_info);
2608 }
2609
2610 /* Returns TRUE if scan is finished. */
2611
2612 static int pqi_scan_finished(struct Scsi_Host *shost,
2613         unsigned long elapsed_time)
2614 {
2615         struct pqi_ctrl_info *ctrl_info;
2616
2617         ctrl_info = shost_priv(shost);
2618
2619         return !mutex_is_locked(&ctrl_info->scan_mutex);
2620 }
2621
2622 static inline void pqi_set_encryption_info(struct pqi_encryption_info *encryption_info,
2623         struct raid_map *raid_map, u64 first_block)
2624 {
2625         u32 volume_blk_size;
2626
2627         /*
2628          * Set the encryption tweak values based on logical block address.
2629          * If the block size is 512, the tweak value is equal to the LBA.
2630          * For other block sizes, tweak value is (LBA * block size) / 512.
2631          */
2632         volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
2633         if (volume_blk_size != 512)
2634                 first_block = (first_block * volume_blk_size) / 512;
2635
2636         encryption_info->data_encryption_key_index =
2637                 get_unaligned_le16(&raid_map->data_encryption_key_index);
2638         encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
2639         encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
2640 }
2641
2642 /*
2643  * Attempt to perform RAID bypass mapping for a logical volume I/O.
2644  */
2645
2646 static bool pqi_aio_raid_level_supported(struct pqi_ctrl_info *ctrl_info,
2647         struct pqi_scsi_dev_raid_map_data *rmd)
2648 {
2649         bool is_supported = true;
2650
2651         switch (rmd->raid_level) {
2652         case SA_RAID_0:
2653                 break;
2654         case SA_RAID_1:
2655                 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2656                         rmd->data_length > ctrl_info->max_write_raid_1_10_2drive))
2657                         is_supported = false;
2658                 break;
2659         case SA_RAID_TRIPLE:
2660                 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2661                         rmd->data_length > ctrl_info->max_write_raid_1_10_3drive))
2662                         is_supported = false;
2663                 break;
2664         case SA_RAID_5:
2665                 if (rmd->is_write && (!ctrl_info->enable_r5_writes ||
2666                         rmd->data_length > ctrl_info->max_write_raid_5_6))
2667                         is_supported = false;
2668                 break;
2669         case SA_RAID_6:
2670                 if (rmd->is_write && (!ctrl_info->enable_r6_writes ||
2671                         rmd->data_length > ctrl_info->max_write_raid_5_6))
2672                         is_supported = false;
2673                 break;
2674         default:
2675                 is_supported = false;
2676                 break;
2677         }
2678
2679         return is_supported;
2680 }
2681
2682 #define PQI_RAID_BYPASS_INELIGIBLE      1
2683
2684 static int pqi_get_aio_lba_and_block_count(struct scsi_cmnd *scmd,
2685         struct pqi_scsi_dev_raid_map_data *rmd)
2686 {
2687         /* Check for valid opcode, get LBA and block count. */
2688         switch (scmd->cmnd[0]) {
2689         case WRITE_6:
2690                 rmd->is_write = true;
2691                 fallthrough;
2692         case READ_6:
2693                 rmd->first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
2694                         (scmd->cmnd[2] << 8) | scmd->cmnd[3]);
2695                 rmd->block_cnt = (u32)scmd->cmnd[4];
2696                 if (rmd->block_cnt == 0)
2697                         rmd->block_cnt = 256;
2698                 break;
2699         case WRITE_10:
2700                 rmd->is_write = true;
2701                 fallthrough;
2702         case READ_10:
2703                 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2704                 rmd->block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
2705                 break;
2706         case WRITE_12:
2707                 rmd->is_write = true;
2708                 fallthrough;
2709         case READ_12:
2710                 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2711                 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
2712                 break;
2713         case WRITE_16:
2714                 rmd->is_write = true;
2715                 fallthrough;
2716         case READ_16:
2717                 rmd->first_block = get_unaligned_be64(&scmd->cmnd[2]);
2718                 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
2719                 break;
2720         default:
2721                 /* Process via normal I/O path. */
2722                 return PQI_RAID_BYPASS_INELIGIBLE;
2723         }
2724
2725         put_unaligned_le32(scsi_bufflen(scmd), &rmd->data_length);
2726
2727         return 0;
2728 }
2729
2730 static int pci_get_aio_common_raid_map_values(struct pqi_ctrl_info *ctrl_info,
2731         struct pqi_scsi_dev_raid_map_data *rmd, struct raid_map *raid_map)
2732 {
2733 #if BITS_PER_LONG == 32
2734         u64 tmpdiv;
2735 #endif
2736
2737         rmd->last_block = rmd->first_block + rmd->block_cnt - 1;
2738
2739         /* Check for invalid block or wraparound. */
2740         if (rmd->last_block >=
2741                 get_unaligned_le64(&raid_map->volume_blk_cnt) ||
2742                 rmd->last_block < rmd->first_block)
2743                 return PQI_RAID_BYPASS_INELIGIBLE;
2744
2745         rmd->data_disks_per_row =
2746                 get_unaligned_le16(&raid_map->data_disks_per_row);
2747         rmd->strip_size = get_unaligned_le16(&raid_map->strip_size);
2748         rmd->layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
2749
2750         /* Calculate stripe information for the request. */
2751         rmd->blocks_per_row = rmd->data_disks_per_row * rmd->strip_size;
2752         if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2753                 return PQI_RAID_BYPASS_INELIGIBLE;
2754 #if BITS_PER_LONG == 32
2755         tmpdiv = rmd->first_block;
2756         do_div(tmpdiv, rmd->blocks_per_row);
2757         rmd->first_row = tmpdiv;
2758         tmpdiv = rmd->last_block;
2759         do_div(tmpdiv, rmd->blocks_per_row);
2760         rmd->last_row = tmpdiv;
2761         rmd->first_row_offset = (u32)(rmd->first_block - (rmd->first_row * rmd->blocks_per_row));
2762         rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row * rmd->blocks_per_row));
2763         tmpdiv = rmd->first_row_offset;
2764         do_div(tmpdiv, rmd->strip_size);
2765         rmd->first_column = tmpdiv;
2766         tmpdiv = rmd->last_row_offset;
2767         do_div(tmpdiv, rmd->strip_size);
2768         rmd->last_column = tmpdiv;
2769 #else
2770         rmd->first_row = rmd->first_block / rmd->blocks_per_row;
2771         rmd->last_row = rmd->last_block / rmd->blocks_per_row;
2772         rmd->first_row_offset = (u32)(rmd->first_block -
2773                 (rmd->first_row * rmd->blocks_per_row));
2774         rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row *
2775                 rmd->blocks_per_row));
2776         rmd->first_column = rmd->first_row_offset / rmd->strip_size;
2777         rmd->last_column = rmd->last_row_offset / rmd->strip_size;
2778 #endif
2779
2780         /* If this isn't a single row/column then give to the controller. */
2781         if (rmd->first_row != rmd->last_row ||
2782                 rmd->first_column != rmd->last_column)
2783                 return PQI_RAID_BYPASS_INELIGIBLE;
2784
2785         /* Proceeding with driver mapping. */
2786         rmd->total_disks_per_row = rmd->data_disks_per_row +
2787                 get_unaligned_le16(&raid_map->metadata_disks_per_row);
2788         rmd->map_row = ((u32)(rmd->first_row >>
2789                 raid_map->parity_rotation_shift)) %
2790                 get_unaligned_le16(&raid_map->row_cnt);
2791         rmd->map_index = (rmd->map_row * rmd->total_disks_per_row) +
2792                 rmd->first_column;
2793
2794         return 0;
2795 }
2796
2797 static int pqi_calc_aio_r5_or_r6(struct pqi_scsi_dev_raid_map_data *rmd,
2798         struct raid_map *raid_map)
2799 {
2800 #if BITS_PER_LONG == 32
2801         u64 tmpdiv;
2802 #endif
2803
2804         if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2805                 return PQI_RAID_BYPASS_INELIGIBLE;
2806
2807         /* RAID 50/60 */
2808         /* Verify first and last block are in same RAID group. */
2809         rmd->stripesize = rmd->blocks_per_row * rmd->layout_map_count;
2810 #if BITS_PER_LONG == 32
2811         tmpdiv = rmd->first_block;
2812         rmd->first_group = do_div(tmpdiv, rmd->stripesize);
2813         tmpdiv = rmd->first_group;
2814         do_div(tmpdiv, rmd->blocks_per_row);
2815         rmd->first_group = tmpdiv;
2816         tmpdiv = rmd->last_block;
2817         rmd->last_group = do_div(tmpdiv, rmd->stripesize);
2818         tmpdiv = rmd->last_group;
2819         do_div(tmpdiv, rmd->blocks_per_row);
2820         rmd->last_group = tmpdiv;
2821 #else
2822         rmd->first_group = (rmd->first_block % rmd->stripesize) / rmd->blocks_per_row;
2823         rmd->last_group = (rmd->last_block % rmd->stripesize) / rmd->blocks_per_row;
2824 #endif
2825         if (rmd->first_group != rmd->last_group)
2826                 return PQI_RAID_BYPASS_INELIGIBLE;
2827
2828         /* Verify request is in a single row of RAID 5/6. */
2829 #if BITS_PER_LONG == 32
2830         tmpdiv = rmd->first_block;
2831         do_div(tmpdiv, rmd->stripesize);
2832         rmd->first_row = tmpdiv;
2833         rmd->r5or6_first_row = tmpdiv;
2834         tmpdiv = rmd->last_block;
2835         do_div(tmpdiv, rmd->stripesize);
2836         rmd->r5or6_last_row = tmpdiv;
2837 #else
2838         rmd->first_row = rmd->r5or6_first_row =
2839                 rmd->first_block / rmd->stripesize;
2840         rmd->r5or6_last_row = rmd->last_block / rmd->stripesize;
2841 #endif
2842         if (rmd->r5or6_first_row != rmd->r5or6_last_row)
2843                 return PQI_RAID_BYPASS_INELIGIBLE;
2844
2845         /* Verify request is in a single column. */
2846 #if BITS_PER_LONG == 32
2847         tmpdiv = rmd->first_block;
2848         rmd->first_row_offset = do_div(tmpdiv, rmd->stripesize);
2849         tmpdiv = rmd->first_row_offset;
2850         rmd->first_row_offset = (u32)do_div(tmpdiv, rmd->blocks_per_row);
2851         rmd->r5or6_first_row_offset = rmd->first_row_offset;
2852         tmpdiv = rmd->last_block;
2853         rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->stripesize);
2854         tmpdiv = rmd->r5or6_last_row_offset;
2855         rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->blocks_per_row);
2856         tmpdiv = rmd->r5or6_first_row_offset;
2857         do_div(tmpdiv, rmd->strip_size);
2858         rmd->first_column = rmd->r5or6_first_column = tmpdiv;
2859         tmpdiv = rmd->r5or6_last_row_offset;
2860         do_div(tmpdiv, rmd->strip_size);
2861         rmd->r5or6_last_column = tmpdiv;
2862 #else
2863         rmd->first_row_offset = rmd->r5or6_first_row_offset =
2864                 (u32)((rmd->first_block % rmd->stripesize) %
2865                 rmd->blocks_per_row);
2866
2867         rmd->r5or6_last_row_offset =
2868                 (u32)((rmd->last_block % rmd->stripesize) %
2869                 rmd->blocks_per_row);
2870
2871         rmd->first_column =
2872                 rmd->r5or6_first_row_offset / rmd->strip_size;
2873         rmd->r5or6_first_column = rmd->first_column;
2874         rmd->r5or6_last_column = rmd->r5or6_last_row_offset / rmd->strip_size;
2875 #endif
2876         if (rmd->r5or6_first_column != rmd->r5or6_last_column)
2877                 return PQI_RAID_BYPASS_INELIGIBLE;
2878
2879         /* Request is eligible. */
2880         rmd->map_row =
2881                 ((u32)(rmd->first_row >> raid_map->parity_rotation_shift)) %
2882                 get_unaligned_le16(&raid_map->row_cnt);
2883
2884         rmd->map_index = (rmd->first_group *
2885                 (get_unaligned_le16(&raid_map->row_cnt) *
2886                 rmd->total_disks_per_row)) +
2887                 (rmd->map_row * rmd->total_disks_per_row) + rmd->first_column;
2888
2889         if (rmd->is_write) {
2890                 u32 index;
2891
2892                 /*
2893                  * p_parity_it_nexus and q_parity_it_nexus are pointers to the
2894                  * parity entries inside the device's raid_map.
2895                  *
2896                  * A device's RAID map is bounded by: number of RAID disks squared.
2897                  *
2898                  * The devices RAID map size is checked during device
2899                  * initialization.
2900                  */
2901                 index = DIV_ROUND_UP(rmd->map_index + 1, rmd->total_disks_per_row);
2902                 index *= rmd->total_disks_per_row;
2903                 index -= get_unaligned_le16(&raid_map->metadata_disks_per_row);
2904
2905                 rmd->p_parity_it_nexus = raid_map->disk_data[index].aio_handle;
2906                 if (rmd->raid_level == SA_RAID_6) {
2907                         rmd->q_parity_it_nexus = raid_map->disk_data[index + 1].aio_handle;
2908                         rmd->xor_mult = raid_map->disk_data[rmd->map_index].xor_mult[1];
2909                 }
2910 #if BITS_PER_LONG == 32
2911                 tmpdiv = rmd->first_block;
2912                 do_div(tmpdiv, rmd->blocks_per_row);
2913                 rmd->row = tmpdiv;
2914 #else
2915                 rmd->row = rmd->first_block / rmd->blocks_per_row;
2916 #endif
2917         }
2918
2919         return 0;
2920 }
2921
2922 static void pqi_set_aio_cdb(struct pqi_scsi_dev_raid_map_data *rmd)
2923 {
2924         /* Build the new CDB for the physical disk I/O. */
2925         if (rmd->disk_block > 0xffffffff) {
2926                 rmd->cdb[0] = rmd->is_write ? WRITE_16 : READ_16;
2927                 rmd->cdb[1] = 0;
2928                 put_unaligned_be64(rmd->disk_block, &rmd->cdb[2]);
2929                 put_unaligned_be32(rmd->disk_block_cnt, &rmd->cdb[10]);
2930                 rmd->cdb[14] = 0;
2931                 rmd->cdb[15] = 0;
2932                 rmd->cdb_length = 16;
2933         } else {
2934                 rmd->cdb[0] = rmd->is_write ? WRITE_10 : READ_10;
2935                 rmd->cdb[1] = 0;
2936                 put_unaligned_be32((u32)rmd->disk_block, &rmd->cdb[2]);
2937                 rmd->cdb[6] = 0;
2938                 put_unaligned_be16((u16)rmd->disk_block_cnt, &rmd->cdb[7]);
2939                 rmd->cdb[9] = 0;
2940                 rmd->cdb_length = 10;
2941         }
2942 }
2943
2944 static void pqi_calc_aio_r1_nexus(struct raid_map *raid_map,
2945         struct pqi_scsi_dev_raid_map_data *rmd)
2946 {
2947         u32 index;
2948         u32 group;
2949
2950         group = rmd->map_index / rmd->data_disks_per_row;
2951
2952         index = rmd->map_index - (group * rmd->data_disks_per_row);
2953         rmd->it_nexus[0] = raid_map->disk_data[index].aio_handle;
2954         index += rmd->data_disks_per_row;
2955         rmd->it_nexus[1] = raid_map->disk_data[index].aio_handle;
2956         if (rmd->layout_map_count > 2) {
2957                 index += rmd->data_disks_per_row;
2958                 rmd->it_nexus[2] = raid_map->disk_data[index].aio_handle;
2959         }
2960
2961         rmd->num_it_nexus_entries = rmd->layout_map_count;
2962 }
2963
2964 static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
2965         struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
2966         struct pqi_queue_group *queue_group)
2967 {
2968         int rc;
2969         struct raid_map *raid_map;
2970         u32 group;
2971         u32 next_bypass_group;
2972         struct pqi_encryption_info *encryption_info_ptr;
2973         struct pqi_encryption_info encryption_info;
2974         struct pqi_scsi_dev_raid_map_data rmd = { 0 };
2975
2976         rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
2977         if (rc)
2978                 return PQI_RAID_BYPASS_INELIGIBLE;
2979
2980         rmd.raid_level = device->raid_level;
2981
2982         if (!pqi_aio_raid_level_supported(ctrl_info, &rmd))
2983                 return PQI_RAID_BYPASS_INELIGIBLE;
2984
2985         if (unlikely(rmd.block_cnt == 0))
2986                 return PQI_RAID_BYPASS_INELIGIBLE;
2987
2988         raid_map = device->raid_map;
2989
2990         rc = pci_get_aio_common_raid_map_values(ctrl_info, &rmd, raid_map);
2991         if (rc)
2992                 return PQI_RAID_BYPASS_INELIGIBLE;
2993
2994         if (device->raid_level == SA_RAID_1 ||
2995                 device->raid_level == SA_RAID_TRIPLE) {
2996                 if (rmd.is_write) {
2997                         pqi_calc_aio_r1_nexus(raid_map, &rmd);
2998                 } else {
2999                         group = device->next_bypass_group[rmd.map_index];
3000                         next_bypass_group = group + 1;
3001                         if (next_bypass_group >= rmd.layout_map_count)
3002                                 next_bypass_group = 0;
3003                         device->next_bypass_group[rmd.map_index] = next_bypass_group;
3004                         rmd.map_index += group * rmd.data_disks_per_row;
3005                 }
3006         } else if ((device->raid_level == SA_RAID_5 ||
3007                 device->raid_level == SA_RAID_6) &&
3008                 (rmd.layout_map_count > 1 || rmd.is_write)) {
3009                 rc = pqi_calc_aio_r5_or_r6(&rmd, raid_map);
3010                 if (rc)
3011                         return PQI_RAID_BYPASS_INELIGIBLE;
3012         }
3013
3014         if (unlikely(rmd.map_index >= RAID_MAP_MAX_ENTRIES))
3015                 return PQI_RAID_BYPASS_INELIGIBLE;
3016
3017         rmd.aio_handle = raid_map->disk_data[rmd.map_index].aio_handle;
3018         rmd.disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
3019                 rmd.first_row * rmd.strip_size +
3020                 (rmd.first_row_offset - rmd.first_column * rmd.strip_size);
3021         rmd.disk_block_cnt = rmd.block_cnt;
3022
3023         /* Handle differing logical/physical block sizes. */
3024         if (raid_map->phys_blk_shift) {
3025                 rmd.disk_block <<= raid_map->phys_blk_shift;
3026                 rmd.disk_block_cnt <<= raid_map->phys_blk_shift;
3027         }
3028
3029         if (unlikely(rmd.disk_block_cnt > 0xffff))
3030                 return PQI_RAID_BYPASS_INELIGIBLE;
3031
3032         pqi_set_aio_cdb(&rmd);
3033
3034         if (get_unaligned_le16(&raid_map->flags) & RAID_MAP_ENCRYPTION_ENABLED) {
3035                 if (rmd.data_length > device->max_transfer_encrypted)
3036                         return PQI_RAID_BYPASS_INELIGIBLE;
3037                 pqi_set_encryption_info(&encryption_info, raid_map, rmd.first_block);
3038                 encryption_info_ptr = &encryption_info;
3039         } else {
3040                 encryption_info_ptr = NULL;
3041         }
3042
3043         if (rmd.is_write) {
3044                 switch (device->raid_level) {
3045                 case SA_RAID_1:
3046                 case SA_RAID_TRIPLE:
3047                         return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,
3048                                 encryption_info_ptr, device, &rmd);
3049                 case SA_RAID_5:
3050                 case SA_RAID_6:
3051                         return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,
3052                                 encryption_info_ptr, device, &rmd);
3053                 }
3054         }
3055
3056         return pqi_aio_submit_io(ctrl_info, scmd, rmd.aio_handle,
3057                 rmd.cdb, rmd.cdb_length, queue_group,
3058                 encryption_info_ptr, true, false);
3059 }
3060
3061 #define PQI_STATUS_IDLE         0x0
3062
3063 #define PQI_CREATE_ADMIN_QUEUE_PAIR     1
3064 #define PQI_DELETE_ADMIN_QUEUE_PAIR     2
3065
3066 #define PQI_DEVICE_STATE_POWER_ON_AND_RESET             0x0
3067 #define PQI_DEVICE_STATE_STATUS_AVAILABLE               0x1
3068 #define PQI_DEVICE_STATE_ALL_REGISTERS_READY            0x2
3069 #define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY         0x3
3070 #define PQI_DEVICE_STATE_ERROR                          0x4
3071
3072 #define PQI_MODE_READY_TIMEOUT_SECS             30
3073 #define PQI_MODE_READY_POLL_INTERVAL_MSECS      1
3074
3075 static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
3076 {
3077         struct pqi_device_registers __iomem *pqi_registers;
3078         unsigned long timeout;
3079         u64 signature;
3080         u8 status;
3081
3082         pqi_registers = ctrl_info->pqi_registers;
3083         timeout = (PQI_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
3084
3085         while (1) {
3086                 signature = readq(&pqi_registers->signature);
3087                 if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
3088                         sizeof(signature)) == 0)
3089                         break;
3090                 if (time_after(jiffies, timeout)) {
3091                         dev_err(&ctrl_info->pci_dev->dev,
3092                                 "timed out waiting for PQI signature\n");
3093                         return -ETIMEDOUT;
3094                 }
3095                 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3096         }
3097
3098         while (1) {
3099                 status = readb(&pqi_registers->function_and_status_code);
3100                 if (status == PQI_STATUS_IDLE)
3101                         break;
3102                 if (time_after(jiffies, timeout)) {
3103                         dev_err(&ctrl_info->pci_dev->dev,
3104                                 "timed out waiting for PQI IDLE\n");
3105                         return -ETIMEDOUT;
3106                 }
3107                 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3108         }
3109
3110         while (1) {
3111                 if (readl(&pqi_registers->device_status) ==
3112                         PQI_DEVICE_STATE_ALL_REGISTERS_READY)
3113                         break;
3114                 if (time_after(jiffies, timeout)) {
3115                         dev_err(&ctrl_info->pci_dev->dev,
3116                                 "timed out waiting for PQI all registers ready\n");
3117                         return -ETIMEDOUT;
3118                 }
3119                 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3120         }
3121
3122         return 0;
3123 }
3124
3125 static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
3126 {
3127         struct pqi_scsi_dev *device;
3128
3129         device = io_request->scmd->device->hostdata;
3130         device->raid_bypass_enabled = false;
3131         device->aio_enabled = false;
3132 }
3133
3134 static inline void pqi_take_device_offline(struct scsi_device *sdev, char *path)
3135 {
3136         struct pqi_ctrl_info *ctrl_info;
3137         struct pqi_scsi_dev *device;
3138
3139         device = sdev->hostdata;
3140         if (device->device_offline)
3141                 return;
3142
3143         device->device_offline = true;
3144         ctrl_info = shost_to_hba(sdev->host);
3145         pqi_schedule_rescan_worker(ctrl_info);
3146         dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",
3147                 path, ctrl_info->scsi_host->host_no, device->bus,
3148                 device->target, device->lun);
3149 }
3150
3151 static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
3152 {
3153         u8 scsi_status;
3154         u8 host_byte;
3155         struct scsi_cmnd *scmd;
3156         struct pqi_raid_error_info *error_info;
3157         size_t sense_data_length;
3158         int residual_count;
3159         int xfer_count;
3160         struct scsi_sense_hdr sshdr;
3161
3162         scmd = io_request->scmd;
3163         if (!scmd)
3164                 return;
3165
3166         error_info = io_request->error_info;
3167         scsi_status = error_info->status;
3168         host_byte = DID_OK;
3169
3170         switch (error_info->data_out_result) {
3171         case PQI_DATA_IN_OUT_GOOD:
3172                 break;
3173         case PQI_DATA_IN_OUT_UNDERFLOW:
3174                 xfer_count =
3175                         get_unaligned_le32(&error_info->data_out_transferred);
3176                 residual_count = scsi_bufflen(scmd) - xfer_count;
3177                 scsi_set_resid(scmd, residual_count);
3178                 if (xfer_count < scmd->underflow)
3179                         host_byte = DID_SOFT_ERROR;
3180                 break;
3181         case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
3182         case PQI_DATA_IN_OUT_ABORTED:
3183                 host_byte = DID_ABORT;
3184                 break;
3185         case PQI_DATA_IN_OUT_TIMEOUT:
3186                 host_byte = DID_TIME_OUT;
3187                 break;
3188         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
3189         case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
3190         case PQI_DATA_IN_OUT_BUFFER_ERROR:
3191         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
3192         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
3193         case PQI_DATA_IN_OUT_ERROR:
3194         case PQI_DATA_IN_OUT_HARDWARE_ERROR:
3195         case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
3196         case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
3197         case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
3198         case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
3199         case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
3200         case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
3201         case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
3202         case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
3203         case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
3204         default:
3205                 host_byte = DID_ERROR;
3206                 break;
3207         }
3208
3209         sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
3210         if (sense_data_length == 0)
3211                 sense_data_length =
3212                         get_unaligned_le16(&error_info->response_data_length);
3213         if (sense_data_length) {
3214                 if (sense_data_length > sizeof(error_info->data))
3215                         sense_data_length = sizeof(error_info->data);
3216
3217                 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
3218                         scsi_normalize_sense(error_info->data,
3219                                 sense_data_length, &sshdr) &&
3220                                 sshdr.sense_key == HARDWARE_ERROR &&
3221                                 sshdr.asc == 0x3e) {
3222                         struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);
3223                         struct pqi_scsi_dev *device = scmd->device->hostdata;
3224
3225                         switch (sshdr.ascq) {
3226                         case 0x1: /* LOGICAL UNIT FAILURE */
3227                                 if (printk_ratelimit())
3228                                         scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n",
3229                                                 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
3230                                 pqi_take_device_offline(scmd->device, "RAID");
3231                                 host_byte = DID_NO_CONNECT;
3232                                 break;
3233
3234                         default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */
3235                                 if (printk_ratelimit())
3236                                         scmd_printk(KERN_ERR, scmd, "received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
3237                                                 sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
3238                                 break;
3239                         }
3240                 }
3241
3242                 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
3243                         sense_data_length = SCSI_SENSE_BUFFERSIZE;
3244                 memcpy(scmd->sense_buffer, error_info->data,
3245                         sense_data_length);
3246         }
3247
3248         scmd->result = scsi_status;
3249         set_host_byte(scmd, host_byte);
3250 }
3251
3252 static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
3253 {
3254         u8 scsi_status;
3255         u8 host_byte;
3256         struct scsi_cmnd *scmd;
3257         struct pqi_aio_error_info *error_info;
3258         size_t sense_data_length;
3259         int residual_count;
3260         int xfer_count;
3261         bool device_offline;
3262         struct pqi_scsi_dev *device;
3263
3264         scmd = io_request->scmd;
3265         error_info = io_request->error_info;
3266         host_byte = DID_OK;
3267         sense_data_length = 0;
3268         device_offline = false;
3269         device = scmd->device->hostdata;
3270
3271         switch (error_info->service_response) {
3272         case PQI_AIO_SERV_RESPONSE_COMPLETE:
3273                 scsi_status = error_info->status;
3274                 break;
3275         case PQI_AIO_SERV_RESPONSE_FAILURE:
3276                 switch (error_info->status) {
3277                 case PQI_AIO_STATUS_IO_ABORTED:
3278                         scsi_status = SAM_STAT_TASK_ABORTED;
3279                         break;
3280                 case PQI_AIO_STATUS_UNDERRUN:
3281                         scsi_status = SAM_STAT_GOOD;
3282                         residual_count = get_unaligned_le32(
3283                                                 &error_info->residual_count);
3284                         scsi_set_resid(scmd, residual_count);
3285                         xfer_count = scsi_bufflen(scmd) - residual_count;
3286                         if (xfer_count < scmd->underflow)
3287                                 host_byte = DID_SOFT_ERROR;
3288                         break;
3289                 case PQI_AIO_STATUS_OVERRUN:
3290                         scsi_status = SAM_STAT_GOOD;
3291                         break;
3292                 case PQI_AIO_STATUS_AIO_PATH_DISABLED:
3293                         pqi_aio_path_disabled(io_request);
3294                         if (pqi_is_multipath_device(device)) {
3295                                 pqi_device_remove_start(device);
3296                                 host_byte = DID_NO_CONNECT;
3297                                 scsi_status = SAM_STAT_CHECK_CONDITION;
3298                         } else {
3299                                 scsi_status = SAM_STAT_GOOD;
3300                                 io_request->status = -EAGAIN;
3301                         }
3302                         break;
3303                 case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
3304                 case PQI_AIO_STATUS_INVALID_DEVICE:
3305                         if (!io_request->raid_bypass) {
3306                                 device_offline = true;
3307                                 pqi_take_device_offline(scmd->device, "AIO");
3308                                 host_byte = DID_NO_CONNECT;
3309                         }
3310                         scsi_status = SAM_STAT_CHECK_CONDITION;
3311                         break;
3312                 case PQI_AIO_STATUS_IO_ERROR:
3313                 default:
3314                         scsi_status = SAM_STAT_CHECK_CONDITION;
3315                         break;
3316                 }
3317                 break;
3318         case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
3319         case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
3320                 scsi_status = SAM_STAT_GOOD;
3321                 break;
3322         case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
3323         case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
3324         default:
3325                 scsi_status = SAM_STAT_CHECK_CONDITION;
3326                 break;
3327         }
3328
3329         if (error_info->data_present) {
3330                 sense_data_length =
3331                         get_unaligned_le16(&error_info->data_length);
3332                 if (sense_data_length) {
3333                         if (sense_data_length > sizeof(error_info->data))
3334                                 sense_data_length = sizeof(error_info->data);
3335                         if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
3336                                 sense_data_length = SCSI_SENSE_BUFFERSIZE;
3337                         memcpy(scmd->sense_buffer, error_info->data,
3338                                 sense_data_length);
3339                 }
3340         }
3341
3342         if (device_offline && sense_data_length == 0)
3343                 scsi_build_sense(scmd, 0, HARDWARE_ERROR, 0x3e, 0x1);
3344
3345         scmd->result = scsi_status;
3346         set_host_byte(scmd, host_byte);
3347 }
3348
3349 static void pqi_process_io_error(unsigned int iu_type,
3350         struct pqi_io_request *io_request)
3351 {
3352         switch (iu_type) {
3353         case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3354                 pqi_process_raid_io_error(io_request);
3355                 break;
3356         case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3357                 pqi_process_aio_io_error(io_request);
3358                 break;
3359         }
3360 }
3361
3362 static int pqi_interpret_task_management_response(struct pqi_ctrl_info *ctrl_info,
3363         struct pqi_task_management_response *response)
3364 {
3365         int rc;
3366
3367         switch (response->response_code) {
3368         case SOP_TMF_COMPLETE:
3369         case SOP_TMF_FUNCTION_SUCCEEDED:
3370                 rc = 0;
3371                 break;
3372         case SOP_TMF_REJECTED:
3373                 rc = -EAGAIN;
3374                 break;
3375         case SOP_TMF_INCORRECT_LOGICAL_UNIT:
3376                 rc = -ENODEV;
3377                 break;
3378         default:
3379                 rc = -EIO;
3380                 break;
3381         }
3382
3383         if (rc)
3384                 dev_err(&ctrl_info->pci_dev->dev,
3385                         "Task Management Function error: %d (response code: %u)\n", rc, response->response_code);
3386
3387         return rc;
3388 }
3389
3390 static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info,
3391         enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
3392 {
3393         pqi_take_ctrl_offline(ctrl_info, ctrl_shutdown_reason);
3394 }
3395
3396 static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)
3397 {
3398         int num_responses;
3399         pqi_index_t oq_pi;
3400         pqi_index_t oq_ci;
3401         struct pqi_io_request *io_request;
3402         struct pqi_io_response *response;
3403         u16 request_id;
3404
3405         num_responses = 0;
3406         oq_ci = queue_group->oq_ci_copy;
3407
3408         while (1) {
3409                 oq_pi = readl(queue_group->oq_pi);
3410                 if (oq_pi >= ctrl_info->num_elements_per_oq) {
3411                         pqi_invalid_response(ctrl_info, PQI_IO_PI_OUT_OF_RANGE);
3412                         dev_err(&ctrl_info->pci_dev->dev,
3413                                 "I/O interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3414                                 oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);
3415                         return -1;
3416                 }
3417                 if (oq_pi == oq_ci)
3418                         break;
3419
3420                 num_responses++;
3421                 response = queue_group->oq_element_array +
3422                         (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
3423
3424                 request_id = get_unaligned_le16(&response->request_id);
3425                 if (request_id >= ctrl_info->max_io_slots) {
3426                         pqi_invalid_response(ctrl_info, PQI_INVALID_REQ_ID);
3427                         dev_err(&ctrl_info->pci_dev->dev,
3428                                 "request ID in response (%u) out of range (0-%u): producer index: %u  consumer index: %u\n",
3429                                 request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);
3430                         return -1;
3431                 }
3432
3433                 io_request = &ctrl_info->io_request_pool[request_id];
3434                 if (atomic_read(&io_request->refcount) == 0) {
3435                         pqi_invalid_response(ctrl_info, PQI_UNMATCHED_REQ_ID);
3436                         dev_err(&ctrl_info->pci_dev->dev,
3437                                 "request ID in response (%u) does not match an outstanding I/O request: producer index: %u  consumer index: %u\n",
3438                                 request_id, oq_pi, oq_ci);
3439                         return -1;
3440                 }
3441
3442                 switch (response->header.iu_type) {
3443                 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
3444                 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
3445                         if (io_request->scmd)
3446                                 io_request->scmd->result = 0;
3447                         fallthrough;
3448                 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
3449                         break;
3450                 case PQI_RESPONSE_IU_VENDOR_GENERAL:
3451                         io_request->status =
3452                                 get_unaligned_le16(
3453                                 &((struct pqi_vendor_general_response *)response)->status);
3454                         break;
3455                 case PQI_RESPONSE_IU_TASK_MANAGEMENT:
3456                         io_request->status = pqi_interpret_task_management_response(ctrl_info,
3457                                 (void *)response);
3458                         break;
3459                 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
3460                         pqi_aio_path_disabled(io_request);
3461                         io_request->status = -EAGAIN;
3462                         break;
3463                 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3464                 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3465                         io_request->error_info = ctrl_info->error_buffer +
3466                                 (get_unaligned_le16(&response->error_index) *
3467                                 PQI_ERROR_BUFFER_ELEMENT_LENGTH);
3468                         pqi_process_io_error(response->header.iu_type, io_request);
3469                         break;
3470                 default:
3471                         pqi_invalid_response(ctrl_info, PQI_UNEXPECTED_IU_TYPE);
3472                         dev_err(&ctrl_info->pci_dev->dev,
3473                                 "unexpected IU type: 0x%x: producer index: %u  consumer index: %u\n",
3474                                 response->header.iu_type, oq_pi, oq_ci);
3475                         return -1;
3476                 }
3477
3478                 io_request->io_complete_callback(io_request, io_request->context);
3479
3480                 /*
3481                  * Note that the I/O request structure CANNOT BE TOUCHED after
3482                  * returning from the I/O completion callback!
3483                  */
3484                 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
3485         }
3486
3487         if (num_responses) {
3488                 queue_group->oq_ci_copy = oq_ci;
3489                 writel(oq_ci, queue_group->oq_ci);
3490         }
3491
3492         return num_responses;
3493 }
3494
3495 static inline unsigned int pqi_num_elements_free(unsigned int pi,
3496         unsigned int ci, unsigned int elements_in_queue)
3497 {
3498         unsigned int num_elements_used;
3499
3500         if (pi >= ci)
3501                 num_elements_used = pi - ci;
3502         else
3503                 num_elements_used = elements_in_queue - ci + pi;
3504
3505         return elements_in_queue - num_elements_used - 1;
3506 }
3507
3508 static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,
3509         struct pqi_event_acknowledge_request *iu, size_t iu_length)
3510 {
3511         pqi_index_t iq_pi;
3512         pqi_index_t iq_ci;
3513         unsigned long flags;
3514         void *next_element;
3515         struct pqi_queue_group *queue_group;
3516
3517         queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
3518         put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
3519
3520         while (1) {
3521                 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
3522
3523                 iq_pi = queue_group->iq_pi_copy[RAID_PATH];
3524                 iq_ci = readl(queue_group->iq_ci[RAID_PATH]);
3525
3526                 if (pqi_num_elements_free(iq_pi, iq_ci,
3527                         ctrl_info->num_elements_per_iq))
3528                         break;
3529
3530                 spin_unlock_irqrestore(
3531                         &queue_group->submit_lock[RAID_PATH], flags);
3532
3533                 if (pqi_ctrl_offline(ctrl_info))
3534                         return;
3535         }
3536
3537         next_element = queue_group->iq_element_array[RAID_PATH] +
3538                 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3539
3540         memcpy(next_element, iu, iu_length);
3541
3542         iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
3543         queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
3544
3545         /*
3546          * This write notifies the controller that an IU is available to be
3547          * processed.
3548          */
3549         writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
3550
3551         spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
3552 }
3553
3554 static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
3555         struct pqi_event *event)
3556 {
3557         struct pqi_event_acknowledge_request request;
3558
3559         memset(&request, 0, sizeof(request));
3560
3561         request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
3562         put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
3563                 &request.header.iu_length);
3564         request.event_type = event->event_type;
3565         put_unaligned_le16(event->event_id, &request.event_id);
3566         put_unaligned_le32(event->additional_event_id, &request.additional_event_id);
3567
3568         pqi_send_event_ack(ctrl_info, &request, sizeof(request));
3569 }
3570
3571 #define PQI_SOFT_RESET_STATUS_TIMEOUT_SECS              30
3572 #define PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS        1
3573
3574 static enum pqi_soft_reset_status pqi_poll_for_soft_reset_status(
3575         struct pqi_ctrl_info *ctrl_info)
3576 {
3577         u8 status;
3578         unsigned long timeout;
3579
3580         timeout = (PQI_SOFT_RESET_STATUS_TIMEOUT_SECS * HZ) + jiffies;
3581
3582         while (1) {
3583                 status = pqi_read_soft_reset_status(ctrl_info);
3584                 if (status & PQI_SOFT_RESET_INITIATE)
3585                         return RESET_INITIATE_DRIVER;
3586
3587                 if (status & PQI_SOFT_RESET_ABORT)
3588                         return RESET_ABORT;
3589
3590                 if (!sis_is_firmware_running(ctrl_info))
3591                         return RESET_NORESPONSE;
3592
3593                 if (time_after(jiffies, timeout)) {
3594                         dev_warn(&ctrl_info->pci_dev->dev,
3595                                 "timed out waiting for soft reset status\n");
3596                         return RESET_TIMEDOUT;
3597                 }
3598
3599                 ssleep(PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS);
3600         }
3601 }
3602
3603 static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info)
3604 {
3605         int rc;
3606         unsigned int delay_secs;
3607         enum pqi_soft_reset_status reset_status;
3608
3609         if (ctrl_info->soft_reset_handshake_supported)
3610                 reset_status = pqi_poll_for_soft_reset_status(ctrl_info);
3611         else
3612                 reset_status = RESET_INITIATE_FIRMWARE;
3613
3614         delay_secs = PQI_POST_RESET_DELAY_SECS;
3615
3616         switch (reset_status) {
3617         case RESET_TIMEDOUT:
3618                 delay_secs = PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS;
3619                 fallthrough;
3620         case RESET_INITIATE_DRIVER:
3621                 dev_info(&ctrl_info->pci_dev->dev,
3622                                 "Online Firmware Activation: resetting controller\n");
3623                 sis_soft_reset(ctrl_info);
3624                 fallthrough;
3625         case RESET_INITIATE_FIRMWARE:
3626                 ctrl_info->pqi_mode_enabled = false;
3627                 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
3628                 rc = pqi_ofa_ctrl_restart(ctrl_info, delay_secs);
3629                 pqi_ofa_free_host_buffer(ctrl_info);
3630                 pqi_ctrl_ofa_done(ctrl_info);
3631                 dev_info(&ctrl_info->pci_dev->dev,
3632                                 "Online Firmware Activation: %s\n",
3633                                 rc == 0 ? "SUCCESS" : "FAILED");
3634                 break;
3635         case RESET_ABORT:
3636                 dev_info(&ctrl_info->pci_dev->dev,
3637                                 "Online Firmware Activation ABORTED\n");
3638                 if (ctrl_info->soft_reset_handshake_supported)
3639                         pqi_clear_soft_reset_status(ctrl_info);
3640                 pqi_ofa_free_host_buffer(ctrl_info);
3641                 pqi_ctrl_ofa_done(ctrl_info);
3642                 pqi_ofa_ctrl_unquiesce(ctrl_info);
3643                 break;
3644         case RESET_NORESPONSE:
3645                 fallthrough;
3646         default:
3647                 dev_err(&ctrl_info->pci_dev->dev,
3648                         "unexpected Online Firmware Activation reset status: 0x%x\n",
3649                         reset_status);
3650                 pqi_ofa_free_host_buffer(ctrl_info);
3651                 pqi_ctrl_ofa_done(ctrl_info);
3652                 pqi_ofa_ctrl_unquiesce(ctrl_info);
3653                 pqi_take_ctrl_offline(ctrl_info, PQI_OFA_RESPONSE_TIMEOUT);
3654                 break;
3655         }
3656 }
3657
3658 static void pqi_ofa_memory_alloc_worker(struct work_struct *work)
3659 {
3660         struct pqi_ctrl_info *ctrl_info;
3661
3662         ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_memory_alloc_work);
3663
3664         pqi_ctrl_ofa_start(ctrl_info);
3665         pqi_ofa_setup_host_buffer(ctrl_info);
3666         pqi_ofa_host_memory_update(ctrl_info);
3667 }
3668
3669 static void pqi_ofa_quiesce_worker(struct work_struct *work)
3670 {
3671         struct pqi_ctrl_info *ctrl_info;
3672         struct pqi_event *event;
3673
3674         ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_quiesce_work);
3675
3676         event = &ctrl_info->events[pqi_event_type_to_event_index(PQI_EVENT_TYPE_OFA)];
3677
3678         pqi_ofa_ctrl_quiesce(ctrl_info);
3679         pqi_acknowledge_event(ctrl_info, event);
3680         pqi_process_soft_reset(ctrl_info);
3681 }
3682
3683 static bool pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,
3684         struct pqi_event *event)
3685 {
3686         bool ack_event;
3687
3688         ack_event = true;
3689
3690         switch (event->event_id) {
3691         case PQI_EVENT_OFA_MEMORY_ALLOCATION:
3692                 dev_info(&ctrl_info->pci_dev->dev,
3693                         "received Online Firmware Activation memory allocation request\n");
3694                 schedule_work(&ctrl_info->ofa_memory_alloc_work);
3695                 break;
3696         case PQI_EVENT_OFA_QUIESCE:
3697                 dev_info(&ctrl_info->pci_dev->dev,
3698                         "received Online Firmware Activation quiesce request\n");
3699                 schedule_work(&ctrl_info->ofa_quiesce_work);
3700                 ack_event = false;
3701                 break;
3702         case PQI_EVENT_OFA_CANCELED:
3703                 dev_info(&ctrl_info->pci_dev->dev,
3704                         "received Online Firmware Activation cancel request: reason: %u\n",
3705                         ctrl_info->ofa_cancel_reason);
3706                 pqi_ofa_free_host_buffer(ctrl_info);
3707                 pqi_ctrl_ofa_done(ctrl_info);
3708                 break;
3709         default:
3710                 dev_err(&ctrl_info->pci_dev->dev,
3711                         "received unknown Online Firmware Activation request: event ID: %u\n",
3712                         event->event_id);
3713                 break;
3714         }
3715
3716         return ack_event;
3717 }
3718
3719 static void pqi_mark_volumes_for_rescan(struct pqi_ctrl_info *ctrl_info)
3720 {
3721         unsigned long flags;
3722         struct pqi_scsi_dev *device;
3723
3724         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
3725
3726         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
3727                 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK)
3728                         device->rescan = true;
3729         }
3730
3731         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
3732 }
3733
3734 static void pqi_disable_raid_bypass(struct pqi_ctrl_info *ctrl_info)
3735 {
3736         unsigned long flags;
3737         struct pqi_scsi_dev *device;
3738
3739         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
3740
3741         list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
3742                 if (device->raid_bypass_enabled)
3743                         device->raid_bypass_enabled = false;
3744
3745         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
3746 }
3747
3748 static void pqi_event_worker(struct work_struct *work)
3749 {
3750         unsigned int i;
3751         bool rescan_needed;
3752         struct pqi_ctrl_info *ctrl_info;
3753         struct pqi_event *event;
3754         bool ack_event;
3755
3756         ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
3757
3758         pqi_ctrl_busy(ctrl_info);
3759         pqi_wait_if_ctrl_blocked(ctrl_info);
3760         if (pqi_ctrl_offline(ctrl_info))
3761                 goto out;
3762
3763         rescan_needed = false;
3764         event = ctrl_info->events;
3765         for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
3766                 if (event->pending) {
3767                         event->pending = false;
3768                         if (event->event_type == PQI_EVENT_TYPE_OFA) {
3769                                 ack_event = pqi_ofa_process_event(ctrl_info, event);
3770                         } else {
3771                                 ack_event = true;
3772                                 rescan_needed = true;
3773                                 if (event->event_type == PQI_EVENT_TYPE_LOGICAL_DEVICE)
3774                                         pqi_mark_volumes_for_rescan(ctrl_info);
3775                                 else if (event->event_type == PQI_EVENT_TYPE_AIO_STATE_CHANGE)
3776                                         pqi_disable_raid_bypass(ctrl_info);
3777                         }
3778                         if (ack_event)
3779                                 pqi_acknowledge_event(ctrl_info, event);
3780                 }
3781                 event++;
3782         }
3783
3784 #define PQI_RESCAN_WORK_FOR_EVENT_DELAY         (5 * HZ)
3785
3786         if (rescan_needed)
3787                 pqi_schedule_rescan_worker_with_delay(ctrl_info,
3788                         PQI_RESCAN_WORK_FOR_EVENT_DELAY);
3789
3790 out:
3791         pqi_ctrl_unbusy(ctrl_info);
3792 }
3793
3794 #define PQI_HEARTBEAT_TIMER_INTERVAL    (10 * HZ)
3795
3796 static void pqi_heartbeat_timer_handler(struct timer_list *t)
3797 {
3798         int num_interrupts;
3799         u32 heartbeat_count;
3800         struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t, heartbeat_timer);
3801
3802         pqi_check_ctrl_health(ctrl_info);
3803         if (pqi_ctrl_offline(ctrl_info))
3804                 return;
3805
3806         num_interrupts = atomic_read(&ctrl_info->num_interrupts);
3807         heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);
3808
3809         if (num_interrupts == ctrl_info->previous_num_interrupts) {
3810                 if (heartbeat_count == ctrl_info->previous_heartbeat_count) {
3811                         dev_err(&ctrl_info->pci_dev->dev,
3812                                 "no heartbeat detected - last heartbeat count: %u\n",
3813                                 heartbeat_count);
3814                         pqi_take_ctrl_offline(ctrl_info, PQI_NO_HEARTBEAT);
3815                         return;
3816                 }
3817         } else {
3818                 ctrl_info->previous_num_interrupts = num_interrupts;
3819         }
3820
3821         ctrl_info->previous_heartbeat_count = heartbeat_count;
3822         mod_timer(&ctrl_info->heartbeat_timer,
3823                 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
3824 }
3825
3826 static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3827 {
3828         if (!ctrl_info->heartbeat_counter)
3829                 return;
3830
3831         ctrl_info->previous_num_interrupts =
3832                 atomic_read(&ctrl_info->num_interrupts);
3833         ctrl_info->previous_heartbeat_count =
3834                 pqi_read_heartbeat_counter(ctrl_info);
3835
3836         ctrl_info->heartbeat_timer.expires =
3837                 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
3838         add_timer(&ctrl_info->heartbeat_timer);
3839 }
3840
3841 static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3842 {
3843         del_timer_sync(&ctrl_info->heartbeat_timer);
3844 }
3845
3846 static void pqi_ofa_capture_event_payload(struct pqi_ctrl_info *ctrl_info,
3847         struct pqi_event *event, struct pqi_event_response *response)
3848 {
3849         switch (event->event_id) {
3850         case PQI_EVENT_OFA_MEMORY_ALLOCATION:
3851                 ctrl_info->ofa_bytes_requested =
3852                         get_unaligned_le32(&response->data.ofa_memory_allocation.bytes_requested);
3853                 break;
3854         case PQI_EVENT_OFA_CANCELED:
3855                 ctrl_info->ofa_cancel_reason =
3856                         get_unaligned_le16(&response->data.ofa_cancelled.reason);
3857                 break;
3858         }
3859 }
3860
3861 static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
3862 {
3863         int num_events;
3864         pqi_index_t oq_pi;
3865         pqi_index_t oq_ci;
3866         struct pqi_event_queue *event_queue;
3867         struct pqi_event_response *response;
3868         struct pqi_event *event;
3869         int event_index;
3870
3871         event_queue = &ctrl_info->event_queue;
3872         num_events = 0;
3873         oq_ci = event_queue->oq_ci_copy;
3874
3875         while (1) {
3876                 oq_pi = readl(event_queue->oq_pi);
3877                 if (oq_pi >= PQI_NUM_EVENT_QUEUE_ELEMENTS) {
3878                         pqi_invalid_response(ctrl_info, PQI_EVENT_PI_OUT_OF_RANGE);
3879                         dev_err(&ctrl_info->pci_dev->dev,
3880                                 "event interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3881                                 oq_pi, PQI_NUM_EVENT_QUEUE_ELEMENTS - 1, oq_ci);
3882                         return -1;
3883                 }
3884
3885                 if (oq_pi == oq_ci)
3886                         break;
3887
3888                 num_events++;
3889                 response = event_queue->oq_element_array + (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
3890
3891                 event_index = pqi_event_type_to_event_index(response->event_type);
3892
3893                 if (event_index >= 0 && response->request_acknowledge) {
3894                         event = &ctrl_info->events[event_index];
3895                         event->pending = true;
3896                         event->event_type = response->event_type;
3897                         event->event_id = get_unaligned_le16(&response->event_id);
3898                         event->additional_event_id =
3899                                 get_unaligned_le32(&response->additional_event_id);
3900                         if (event->event_type == PQI_EVENT_TYPE_OFA)
3901                                 pqi_ofa_capture_event_payload(ctrl_info, event, response);
3902                 }
3903
3904                 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
3905         }
3906
3907         if (num_events) {
3908                 event_queue->oq_ci_copy = oq_ci;
3909                 writel(oq_ci, event_queue->oq_ci);
3910                 schedule_work(&ctrl_info->event_work);
3911         }
3912
3913         return num_events;
3914 }
3915
3916 #define PQI_LEGACY_INTX_MASK    0x1
3917
3918 static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info, bool enable_intx)
3919 {
3920         u32 intx_mask;
3921         struct pqi_device_registers __iomem *pqi_registers;
3922         volatile void __iomem *register_addr;
3923
3924         pqi_registers = ctrl_info->pqi_registers;
3925
3926         if (enable_intx)
3927                 register_addr = &pqi_registers->legacy_intx_mask_clear;
3928         else
3929                 register_addr = &pqi_registers->legacy_intx_mask_set;
3930
3931         intx_mask = readl(register_addr);
3932         intx_mask |= PQI_LEGACY_INTX_MASK;
3933         writel(intx_mask, register_addr);
3934 }
3935
3936 static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
3937         enum pqi_irq_mode new_mode)
3938 {
3939         switch (ctrl_info->irq_mode) {
3940         case IRQ_MODE_MSIX:
3941                 switch (new_mode) {
3942                 case IRQ_MODE_MSIX:
3943                         break;
3944                 case IRQ_MODE_INTX:
3945                         pqi_configure_legacy_intx(ctrl_info, true);
3946                         sis_enable_intx(ctrl_info);
3947                         break;
3948                 case IRQ_MODE_NONE:
3949                         break;
3950                 }
3951                 break;
3952         case IRQ_MODE_INTX:
3953                 switch (new_mode) {
3954                 case IRQ_MODE_MSIX:
3955                         pqi_configure_legacy_intx(ctrl_info, false);
3956                         sis_enable_msix(ctrl_info);
3957                         break;
3958                 case IRQ_MODE_INTX:
3959                         break;
3960                 case IRQ_MODE_NONE:
3961                         pqi_configure_legacy_intx(ctrl_info, false);
3962                         break;
3963                 }
3964                 break;
3965         case IRQ_MODE_NONE:
3966                 switch (new_mode) {
3967                 case IRQ_MODE_MSIX:
3968                         sis_enable_msix(ctrl_info);
3969                         break;
3970                 case IRQ_MODE_INTX:
3971                         pqi_configure_legacy_intx(ctrl_info, true);
3972                         sis_enable_intx(ctrl_info);
3973                         break;
3974                 case IRQ_MODE_NONE:
3975                         break;
3976                 }
3977                 break;
3978         }
3979
3980         ctrl_info->irq_mode = new_mode;
3981 }
3982
3983 #define PQI_LEGACY_INTX_PENDING         0x1
3984
3985 static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)
3986 {
3987         bool valid_irq;
3988         u32 intx_status;
3989
3990         switch (ctrl_info->irq_mode) {
3991         case IRQ_MODE_MSIX:
3992                 valid_irq = true;
3993                 break;
3994         case IRQ_MODE_INTX:
3995                 intx_status = readl(&ctrl_info->pqi_registers->legacy_intx_status);
3996                 if (intx_status & PQI_LEGACY_INTX_PENDING)
3997                         valid_irq = true;
3998                 else
3999                         valid_irq = false;
4000                 break;
4001         case IRQ_MODE_NONE:
4002         default:
4003                 valid_irq = false;
4004                 break;
4005         }
4006
4007         return valid_irq;
4008 }
4009
4010 static irqreturn_t pqi_irq_handler(int irq, void *data)
4011 {
4012         struct pqi_ctrl_info *ctrl_info;
4013         struct pqi_queue_group *queue_group;
4014         int num_io_responses_handled;
4015         int num_events_handled;
4016
4017         queue_group = data;
4018         ctrl_info = queue_group->ctrl_info;
4019
4020         if (!pqi_is_valid_irq(ctrl_info))
4021                 return IRQ_NONE;
4022
4023         num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
4024         if (num_io_responses_handled < 0)
4025                 goto out;
4026
4027         if (irq == ctrl_info->event_irq) {
4028                 num_events_handled = pqi_process_event_intr(ctrl_info);
4029                 if (num_events_handled < 0)
4030                         goto out;
4031         } else {
4032                 num_events_handled = 0;
4033         }
4034
4035         if (num_io_responses_handled + num_events_handled > 0)
4036                 atomic_inc(&ctrl_info->num_interrupts);
4037
4038         pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
4039         pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
4040
4041 out:
4042         return IRQ_HANDLED;
4043 }
4044
4045 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
4046 {
4047         struct pci_dev *pci_dev = ctrl_info->pci_dev;
4048         int i;
4049         int rc;
4050
4051         ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);
4052
4053         for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
4054                 rc = request_irq(pci_irq_vector(pci_dev, i), pqi_irq_handler, 0,
4055                         DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
4056                 if (rc) {
4057                         dev_err(&pci_dev->dev,
4058                                 "irq %u init failed with error %d\n",
4059                                 pci_irq_vector(pci_dev, i), rc);
4060                         return rc;
4061                 }
4062                 ctrl_info->num_msix_vectors_initialized++;
4063         }
4064
4065         return 0;
4066 }
4067
4068 static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
4069 {
4070         int i;
4071
4072         for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
4073                 free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
4074                         &ctrl_info->queue_groups[i]);
4075
4076         ctrl_info->num_msix_vectors_initialized = 0;
4077 }
4078
4079 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
4080 {
4081         int num_vectors_enabled;
4082         unsigned int flags = PCI_IRQ_MSIX;
4083
4084         if (!pqi_disable_managed_interrupts)
4085                 flags |= PCI_IRQ_AFFINITY;
4086
4087         num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,
4088                         PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
4089                         flags);
4090         if (num_vectors_enabled < 0) {
4091                 dev_err(&ctrl_info->pci_dev->dev,
4092                         "MSI-X init failed with error %d\n",
4093                         num_vectors_enabled);
4094                 return num_vectors_enabled;
4095         }
4096
4097         ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
4098         ctrl_info->irq_mode = IRQ_MODE_MSIX;
4099         return 0;
4100 }
4101
4102 static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
4103 {
4104         if (ctrl_info->num_msix_vectors_enabled) {
4105                 pci_free_irq_vectors(ctrl_info->pci_dev);
4106                 ctrl_info->num_msix_vectors_enabled = 0;
4107         }
4108 }
4109
4110 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
4111 {
4112         unsigned int i;
4113         size_t alloc_length;
4114         size_t element_array_length_per_iq;
4115         size_t element_array_length_per_oq;
4116         void *element_array;
4117         void __iomem *next_queue_index;
4118         void *aligned_pointer;
4119         unsigned int num_inbound_queues;
4120         unsigned int num_outbound_queues;
4121         unsigned int num_queue_indexes;
4122         struct pqi_queue_group *queue_group;
4123
4124         element_array_length_per_iq =
4125                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
4126                 ctrl_info->num_elements_per_iq;
4127         element_array_length_per_oq =
4128                 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
4129                 ctrl_info->num_elements_per_oq;
4130         num_inbound_queues = ctrl_info->num_queue_groups * 2;
4131         num_outbound_queues = ctrl_info->num_queue_groups;
4132         num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
4133
4134         aligned_pointer = NULL;
4135
4136         for (i = 0; i < num_inbound_queues; i++) {
4137                 aligned_pointer = PTR_ALIGN(aligned_pointer,
4138                         PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4139                 aligned_pointer += element_array_length_per_iq;
4140         }
4141
4142         for (i = 0; i < num_outbound_queues; i++) {
4143                 aligned_pointer = PTR_ALIGN(aligned_pointer,
4144                         PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4145                 aligned_pointer += element_array_length_per_oq;
4146         }
4147
4148         aligned_pointer = PTR_ALIGN(aligned_pointer,
4149                 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4150         aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
4151                 PQI_EVENT_OQ_ELEMENT_LENGTH;
4152
4153         for (i = 0; i < num_queue_indexes; i++) {
4154                 aligned_pointer = PTR_ALIGN(aligned_pointer,
4155                         PQI_OPERATIONAL_INDEX_ALIGNMENT);
4156                 aligned_pointer += sizeof(pqi_index_t);
4157         }
4158
4159         alloc_length = (size_t)aligned_pointer +
4160                 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
4161
4162         alloc_length += PQI_EXTRA_SGL_MEMORY;
4163
4164         ctrl_info->queue_memory_base =
4165                 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
4166                                    &ctrl_info->queue_memory_base_dma_handle,
4167                                    GFP_KERNEL);
4168
4169         if (!ctrl_info->queue_memory_base)
4170                 return -ENOMEM;
4171
4172         ctrl_info->queue_memory_length = alloc_length;
4173
4174         element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
4175                 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4176
4177         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4178                 queue_group = &ctrl_info->queue_groups[i];
4179                 queue_group->iq_element_array[RAID_PATH] = element_array;
4180                 queue_group->iq_element_array_bus_addr[RAID_PATH] =
4181                         ctrl_info->queue_memory_base_dma_handle +
4182                                 (element_array - ctrl_info->queue_memory_base);
4183                 element_array += element_array_length_per_iq;
4184                 element_array = PTR_ALIGN(element_array,
4185                         PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4186                 queue_group->iq_element_array[AIO_PATH] = element_array;
4187                 queue_group->iq_element_array_bus_addr[AIO_PATH] =
4188                         ctrl_info->queue_memory_base_dma_handle +
4189                         (element_array - ctrl_info->queue_memory_base);
4190                 element_array += element_array_length_per_iq;
4191                 element_array = PTR_ALIGN(element_array,
4192                         PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4193         }
4194
4195         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4196                 queue_group = &ctrl_info->queue_groups[i];
4197                 queue_group->oq_element_array = element_array;
4198                 queue_group->oq_element_array_bus_addr =
4199                         ctrl_info->queue_memory_base_dma_handle +
4200                         (element_array - ctrl_info->queue_memory_base);
4201                 element_array += element_array_length_per_oq;
4202                 element_array = PTR_ALIGN(element_array,
4203                         PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4204         }
4205
4206         ctrl_info->event_queue.oq_element_array = element_array;
4207         ctrl_info->event_queue.oq_element_array_bus_addr =
4208                 ctrl_info->queue_memory_base_dma_handle +
4209                 (element_array - ctrl_info->queue_memory_base);
4210         element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
4211                 PQI_EVENT_OQ_ELEMENT_LENGTH;
4212
4213         next_queue_index = (void __iomem *)PTR_ALIGN(element_array,
4214                 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4215
4216         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4217                 queue_group = &ctrl_info->queue_groups[i];
4218                 queue_group->iq_ci[RAID_PATH] = next_queue_index;
4219                 queue_group->iq_ci_bus_addr[RAID_PATH] =
4220                         ctrl_info->queue_memory_base_dma_handle +
4221                         (next_queue_index -
4222                         (void __iomem *)ctrl_info->queue_memory_base);
4223                 next_queue_index += sizeof(pqi_index_t);
4224                 next_queue_index = PTR_ALIGN(next_queue_index,
4225                         PQI_OPERATIONAL_INDEX_ALIGNMENT);
4226                 queue_group->iq_ci[AIO_PATH] = next_queue_index;
4227                 queue_group->iq_ci_bus_addr[AIO_PATH] =
4228                         ctrl_info->queue_memory_base_dma_handle +
4229                         (next_queue_index -
4230                         (void __iomem *)ctrl_info->queue_memory_base);
4231                 next_queue_index += sizeof(pqi_index_t);
4232                 next_queue_index = PTR_ALIGN(next_queue_index,
4233                         PQI_OPERATIONAL_INDEX_ALIGNMENT);
4234                 queue_group->oq_pi = next_queue_index;
4235                 queue_group->oq_pi_bus_addr =
4236                         ctrl_info->queue_memory_base_dma_handle +
4237                         (next_queue_index -
4238                         (void __iomem *)ctrl_info->queue_memory_base);
4239                 next_queue_index += sizeof(pqi_index_t);
4240                 next_queue_index = PTR_ALIGN(next_queue_index,
4241                         PQI_OPERATIONAL_INDEX_ALIGNMENT);
4242         }
4243
4244         ctrl_info->event_queue.oq_pi = next_queue_index;
4245         ctrl_info->event_queue.oq_pi_bus_addr =
4246                 ctrl_info->queue_memory_base_dma_handle +
4247                 (next_queue_index -
4248                 (void __iomem *)ctrl_info->queue_memory_base);
4249
4250         return 0;
4251 }
4252
4253 static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
4254 {
4255         unsigned int i;
4256         u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
4257         u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
4258
4259         /*
4260          * Initialize the backpointers to the controller structure in
4261          * each operational queue group structure.
4262          */
4263         for (i = 0; i < ctrl_info->num_queue_groups; i++)
4264                 ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
4265
4266         /*
4267          * Assign IDs to all operational queues.  Note that the IDs
4268          * assigned to operational IQs are independent of the IDs
4269          * assigned to operational OQs.
4270          */
4271         ctrl_info->event_queue.oq_id = next_oq_id++;
4272         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4273                 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
4274                 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
4275                 ctrl_info->queue_groups[i].oq_id = next_oq_id++;
4276         }
4277
4278         /*
4279          * Assign MSI-X table entry indexes to all queues.  Note that the
4280          * interrupt for the event queue is shared with the first queue group.
4281          */
4282         ctrl_info->event_queue.int_msg_num = 0;
4283         for (i = 0; i < ctrl_info->num_queue_groups; i++)
4284                 ctrl_info->queue_groups[i].int_msg_num = i;
4285
4286         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4287                 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
4288                 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
4289                 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
4290                 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
4291         }
4292 }
4293
4294 static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
4295 {
4296         size_t alloc_length;
4297         struct pqi_admin_queues_aligned *admin_queues_aligned;
4298         struct pqi_admin_queues *admin_queues;
4299
4300         alloc_length = sizeof(struct pqi_admin_queues_aligned) +
4301                 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
4302
4303         ctrl_info->admin_queue_memory_base =
4304                 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
4305                                    &ctrl_info->admin_queue_memory_base_dma_handle,
4306                                    GFP_KERNEL);
4307
4308         if (!ctrl_info->admin_queue_memory_base)
4309                 return -ENOMEM;
4310
4311         ctrl_info->admin_queue_memory_length = alloc_length;
4312
4313         admin_queues = &ctrl_info->admin_queues;
4314         admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
4315                 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4316         admin_queues->iq_element_array =
4317                 &admin_queues_aligned->iq_element_array;
4318         admin_queues->oq_element_array =
4319                 &admin_queues_aligned->oq_element_array;
4320         admin_queues->iq_ci =
4321                 (pqi_index_t __iomem *)&admin_queues_aligned->iq_ci;
4322         admin_queues->oq_pi =
4323                 (pqi_index_t __iomem *)&admin_queues_aligned->oq_pi;
4324
4325         admin_queues->iq_element_array_bus_addr =
4326                 ctrl_info->admin_queue_memory_base_dma_handle +
4327                 (admin_queues->iq_element_array -
4328                 ctrl_info->admin_queue_memory_base);
4329         admin_queues->oq_element_array_bus_addr =
4330                 ctrl_info->admin_queue_memory_base_dma_handle +
4331                 (admin_queues->oq_element_array -
4332                 ctrl_info->admin_queue_memory_base);
4333         admin_queues->iq_ci_bus_addr =
4334                 ctrl_info->admin_queue_memory_base_dma_handle +
4335                 ((void __iomem *)admin_queues->iq_ci -
4336                 (void __iomem *)ctrl_info->admin_queue_memory_base);
4337         admin_queues->oq_pi_bus_addr =
4338                 ctrl_info->admin_queue_memory_base_dma_handle +
4339                 ((void __iomem *)admin_queues->oq_pi -
4340                 (void __iomem *)ctrl_info->admin_queue_memory_base);
4341
4342         return 0;
4343 }
4344
4345 #define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES          HZ
4346 #define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS      1
4347
4348 static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
4349 {
4350         struct pqi_device_registers __iomem *pqi_registers;
4351         struct pqi_admin_queues *admin_queues;
4352         unsigned long timeout;
4353         u8 status;
4354         u32 reg;
4355
4356         pqi_registers = ctrl_info->pqi_registers;
4357         admin_queues = &ctrl_info->admin_queues;
4358
4359         writeq((u64)admin_queues->iq_element_array_bus_addr,
4360                 &pqi_registers->admin_iq_element_array_addr);
4361         writeq((u64)admin_queues->oq_element_array_bus_addr,
4362                 &pqi_registers->admin_oq_element_array_addr);
4363         writeq((u64)admin_queues->iq_ci_bus_addr,
4364                 &pqi_registers->admin_iq_ci_addr);
4365         writeq((u64)admin_queues->oq_pi_bus_addr,
4366                 &pqi_registers->admin_oq_pi_addr);
4367
4368         reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
4369                 (PQI_ADMIN_OQ_NUM_ELEMENTS << 8) |
4370                 (admin_queues->int_msg_num << 16);
4371         writel(reg, &pqi_registers->admin_iq_num_elements);
4372
4373         writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
4374                 &pqi_registers->function_and_status_code);
4375
4376         timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
4377         while (1) {
4378                 msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
4379                 status = readb(&pqi_registers->function_and_status_code);
4380                 if (status == PQI_STATUS_IDLE)
4381                         break;
4382                 if (time_after(jiffies, timeout))
4383                         return -ETIMEDOUT;
4384         }
4385
4386         /*
4387          * The offset registers are not initialized to the correct
4388          * offsets until *after* the create admin queue pair command
4389          * completes successfully.
4390          */
4391         admin_queues->iq_pi = ctrl_info->iomem_base +
4392                 PQI_DEVICE_REGISTERS_OFFSET +
4393                 readq(&pqi_registers->admin_iq_pi_offset);
4394         admin_queues->oq_ci = ctrl_info->iomem_base +
4395                 PQI_DEVICE_REGISTERS_OFFSET +
4396                 readq(&pqi_registers->admin_oq_ci_offset);
4397
4398         return 0;
4399 }
4400
4401 static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
4402         struct pqi_general_admin_request *request)
4403 {
4404         struct pqi_admin_queues *admin_queues;
4405         void *next_element;
4406         pqi_index_t iq_pi;
4407
4408         admin_queues = &ctrl_info->admin_queues;
4409         iq_pi = admin_queues->iq_pi_copy;
4410
4411         next_element = admin_queues->iq_element_array +
4412                 (iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
4413
4414         memcpy(next_element, request, sizeof(*request));
4415
4416         iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
4417         admin_queues->iq_pi_copy = iq_pi;
4418
4419         /*
4420          * This write notifies the controller that an IU is available to be
4421          * processed.
4422          */
4423         writel(iq_pi, admin_queues->iq_pi);
4424 }
4425
4426 #define PQI_ADMIN_REQUEST_TIMEOUT_SECS  60
4427
4428 static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
4429         struct pqi_general_admin_response *response)
4430 {
4431         struct pqi_admin_queues *admin_queues;
4432         pqi_index_t oq_pi;
4433         pqi_index_t oq_ci;
4434         unsigned long timeout;
4435
4436         admin_queues = &ctrl_info->admin_queues;
4437         oq_ci = admin_queues->oq_ci_copy;
4438
4439         timeout = (PQI_ADMIN_REQUEST_TIMEOUT_SECS * HZ) + jiffies;
4440
4441         while (1) {
4442                 oq_pi = readl(admin_queues->oq_pi);
4443                 if (oq_pi != oq_ci)
4444                         break;
4445                 if (time_after(jiffies, timeout)) {
4446                         dev_err(&ctrl_info->pci_dev->dev,
4447                                 "timed out waiting for admin response\n");
4448                         return -ETIMEDOUT;
4449                 }
4450                 if (!sis_is_firmware_running(ctrl_info))
4451                         return -ENXIO;
4452                 usleep_range(1000, 2000);
4453         }
4454
4455         memcpy(response, admin_queues->oq_element_array +
4456                 (oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
4457
4458         oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
4459         admin_queues->oq_ci_copy = oq_ci;
4460         writel(oq_ci, admin_queues->oq_ci);
4461
4462         return 0;
4463 }
4464
4465 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
4466         struct pqi_queue_group *queue_group, enum pqi_io_path path,
4467         struct pqi_io_request *io_request)
4468 {
4469         struct pqi_io_request *next;
4470         void *next_element;
4471         pqi_index_t iq_pi;
4472         pqi_index_t iq_ci;
4473         size_t iu_length;
4474         unsigned long flags;
4475         unsigned int num_elements_needed;
4476         unsigned int num_elements_to_end_of_queue;
4477         size_t copy_count;
4478         struct pqi_iu_header *request;
4479
4480         spin_lock_irqsave(&queue_group->submit_lock[path], flags);
4481
4482         if (io_request) {
4483                 io_request->queue_group = queue_group;
4484                 list_add_tail(&io_request->request_list_entry,
4485                         &queue_group->request_list[path]);
4486         }
4487
4488         iq_pi = queue_group->iq_pi_copy[path];
4489
4490         list_for_each_entry_safe(io_request, next,
4491                 &queue_group->request_list[path], request_list_entry) {
4492
4493                 request = io_request->iu;
4494
4495                 iu_length = get_unaligned_le16(&request->iu_length) +
4496                         PQI_REQUEST_HEADER_LENGTH;
4497                 num_elements_needed =
4498                         DIV_ROUND_UP(iu_length,
4499                                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4500
4501                 iq_ci = readl(queue_group->iq_ci[path]);
4502
4503                 if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
4504                         ctrl_info->num_elements_per_iq))
4505                         break;
4506
4507                 put_unaligned_le16(queue_group->oq_id,
4508                         &request->response_queue_id);
4509
4510                 next_element = queue_group->iq_element_array[path] +
4511                         (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4512
4513                 num_elements_to_end_of_queue =
4514                         ctrl_info->num_elements_per_iq - iq_pi;
4515
4516                 if (num_elements_needed <= num_elements_to_end_of_queue) {
4517                         memcpy(next_element, request, iu_length);
4518                 } else {
4519                         copy_count = num_elements_to_end_of_queue *
4520                                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
4521                         memcpy(next_element, request, copy_count);
4522                         memcpy(queue_group->iq_element_array[path],
4523                                 (u8 *)request + copy_count,
4524                                 iu_length - copy_count);
4525                 }
4526
4527                 iq_pi = (iq_pi + num_elements_needed) %
4528                         ctrl_info->num_elements_per_iq;
4529
4530                 list_del(&io_request->request_list_entry);
4531         }
4532
4533         if (iq_pi != queue_group->iq_pi_copy[path]) {
4534                 queue_group->iq_pi_copy[path] = iq_pi;
4535                 /*
4536                  * This write notifies the controller that one or more IUs are
4537                  * available to be processed.
4538                  */
4539                 writel(iq_pi, queue_group->iq_pi[path]);
4540         }
4541
4542         spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
4543 }
4544
4545 #define PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS         10
4546
4547 static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,
4548         struct completion *wait)
4549 {
4550         int rc;
4551
4552         while (1) {
4553                 if (wait_for_completion_io_timeout(wait,
4554                         PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS * HZ)) {
4555                         rc = 0;
4556                         break;
4557                 }
4558
4559                 pqi_check_ctrl_health(ctrl_info);
4560                 if (pqi_ctrl_offline(ctrl_info)) {
4561                         rc = -ENXIO;
4562                         break;
4563                 }
4564         }
4565
4566         return rc;
4567 }
4568
4569 static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
4570         void *context)
4571 {
4572         struct completion *waiting = context;
4573
4574         complete(waiting);
4575 }
4576
4577 static int pqi_process_raid_io_error_synchronous(
4578         struct pqi_raid_error_info *error_info)
4579 {
4580         int rc = -EIO;
4581
4582         switch (error_info->data_out_result) {
4583         case PQI_DATA_IN_OUT_GOOD:
4584                 if (error_info->status == SAM_STAT_GOOD)
4585                         rc = 0;
4586                 break;
4587         case PQI_DATA_IN_OUT_UNDERFLOW:
4588                 if (error_info->status == SAM_STAT_GOOD ||
4589                         error_info->status == SAM_STAT_CHECK_CONDITION)
4590                         rc = 0;
4591                 break;
4592         case PQI_DATA_IN_OUT_ABORTED:
4593                 rc = PQI_CMD_STATUS_ABORTED;
4594                 break;
4595         }
4596
4597         return rc;
4598 }
4599
4600 static inline bool pqi_is_blockable_request(struct pqi_iu_header *request)
4601 {
4602         return (request->driver_flags & PQI_DRIVER_NONBLOCKABLE_REQUEST) == 0;
4603 }
4604
4605 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
4606         struct pqi_iu_header *request, unsigned int flags,
4607         struct pqi_raid_error_info *error_info)
4608 {
4609         int rc = 0;
4610         struct pqi_io_request *io_request;
4611         size_t iu_length;
4612         DECLARE_COMPLETION_ONSTACK(wait);
4613
4614         if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
4615                 if (down_interruptible(&ctrl_info->sync_request_sem))
4616                         return -ERESTARTSYS;
4617         } else {
4618                 down(&ctrl_info->sync_request_sem);
4619         }
4620
4621         pqi_ctrl_busy(ctrl_info);
4622         /*
4623          * Wait for other admin queue updates such as;
4624          * config table changes, OFA memory updates, ...
4625          */
4626         if (pqi_is_blockable_request(request))
4627                 pqi_wait_if_ctrl_blocked(ctrl_info);
4628
4629         if (pqi_ctrl_offline(ctrl_info)) {
4630                 rc = -ENXIO;
4631                 goto out;
4632         }
4633
4634         io_request = pqi_alloc_io_request(ctrl_info, NULL);
4635
4636         put_unaligned_le16(io_request->index,
4637                 &(((struct pqi_raid_path_request *)request)->request_id));
4638
4639         if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
4640                 ((struct pqi_raid_path_request *)request)->error_index =
4641                         ((struct pqi_raid_path_request *)request)->request_id;
4642
4643         iu_length = get_unaligned_le16(&request->iu_length) +
4644                 PQI_REQUEST_HEADER_LENGTH;
4645         memcpy(io_request->iu, request, iu_length);
4646
4647         io_request->io_complete_callback = pqi_raid_synchronous_complete;
4648         io_request->context = &wait;
4649
4650         pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
4651                 io_request);
4652
4653         pqi_wait_for_completion_io(ctrl_info, &wait);
4654
4655         if (error_info) {
4656                 if (io_request->error_info)
4657                         memcpy(error_info, io_request->error_info, sizeof(*error_info));
4658                 else
4659                         memset(error_info, 0, sizeof(*error_info));
4660         } else if (rc == 0 && io_request->error_info) {
4661                 rc = pqi_process_raid_io_error_synchronous(io_request->error_info);
4662         }
4663
4664         pqi_free_io_request(io_request);
4665
4666 out:
4667         pqi_ctrl_unbusy(ctrl_info);
4668         up(&ctrl_info->sync_request_sem);
4669
4670         return rc;
4671 }
4672
4673 static int pqi_validate_admin_response(
4674         struct pqi_general_admin_response *response, u8 expected_function_code)
4675 {
4676         if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
4677                 return -EINVAL;
4678
4679         if (get_unaligned_le16(&response->header.iu_length) !=
4680                 PQI_GENERAL_ADMIN_IU_LENGTH)
4681                 return -EINVAL;
4682
4683         if (response->function_code != expected_function_code)
4684                 return -EINVAL;
4685
4686         if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
4687                 return -EINVAL;
4688
4689         return 0;
4690 }
4691
4692 static int pqi_submit_admin_request_synchronous(
4693         struct pqi_ctrl_info *ctrl_info,
4694         struct pqi_general_admin_request *request,
4695         struct pqi_general_admin_response *response)
4696 {
4697         int rc;
4698
4699         pqi_submit_admin_request(ctrl_info, request);
4700
4701         rc = pqi_poll_for_admin_response(ctrl_info, response);
4702
4703         if (rc == 0)
4704                 rc = pqi_validate_admin_response(response, request->function_code);
4705
4706         return rc;
4707 }
4708
4709 static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
4710 {
4711         int rc;
4712         struct pqi_general_admin_request request;
4713         struct pqi_general_admin_response response;
4714         struct pqi_device_capability *capability;
4715         struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
4716
4717         capability = kmalloc(sizeof(*capability), GFP_KERNEL);
4718         if (!capability)
4719                 return -ENOMEM;
4720
4721         memset(&request, 0, sizeof(request));
4722
4723         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4724         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4725                 &request.header.iu_length);
4726         request.function_code =
4727                 PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
4728         put_unaligned_le32(sizeof(*capability),
4729                 &request.data.report_device_capability.buffer_length);
4730
4731         rc = pqi_map_single(ctrl_info->pci_dev,
4732                 &request.data.report_device_capability.sg_descriptor,
4733                 capability, sizeof(*capability),
4734                 DMA_FROM_DEVICE);
4735         if (rc)
4736                 goto out;
4737
4738         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request, &response);
4739
4740         pqi_pci_unmap(ctrl_info->pci_dev,
4741                 &request.data.report_device_capability.sg_descriptor, 1,
4742                 DMA_FROM_DEVICE);
4743
4744         if (rc)
4745                 goto out;
4746
4747         if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
4748                 rc = -EIO;
4749                 goto out;
4750         }
4751
4752         ctrl_info->max_inbound_queues =
4753                 get_unaligned_le16(&capability->max_inbound_queues);
4754         ctrl_info->max_elements_per_iq =
4755                 get_unaligned_le16(&capability->max_elements_per_iq);
4756         ctrl_info->max_iq_element_length =
4757                 get_unaligned_le16(&capability->max_iq_element_length)
4758                 * 16;
4759         ctrl_info->max_outbound_queues =
4760                 get_unaligned_le16(&capability->max_outbound_queues);
4761         ctrl_info->max_elements_per_oq =
4762                 get_unaligned_le16(&capability->max_elements_per_oq);
4763         ctrl_info->max_oq_element_length =
4764                 get_unaligned_le16(&capability->max_oq_element_length)
4765                 * 16;
4766
4767         sop_iu_layer_descriptor =
4768                 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
4769
4770         ctrl_info->max_inbound_iu_length_per_firmware =
4771                 get_unaligned_le16(
4772                         &sop_iu_layer_descriptor->max_inbound_iu_length);
4773         ctrl_info->inbound_spanning_supported =
4774                 sop_iu_layer_descriptor->inbound_spanning_supported;
4775         ctrl_info->outbound_spanning_supported =
4776                 sop_iu_layer_descriptor->outbound_spanning_supported;
4777
4778 out:
4779         kfree(capability);
4780
4781         return rc;
4782 }
4783
4784 static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
4785 {
4786         if (ctrl_info->max_iq_element_length <
4787                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4788                 dev_err(&ctrl_info->pci_dev->dev,
4789                         "max. inbound queue element length of %d is less than the required length of %d\n",
4790                         ctrl_info->max_iq_element_length,
4791                         PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4792                 return -EINVAL;
4793         }
4794
4795         if (ctrl_info->max_oq_element_length <
4796                 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
4797                 dev_err(&ctrl_info->pci_dev->dev,
4798                         "max. outbound queue element length of %d is less than the required length of %d\n",
4799                         ctrl_info->max_oq_element_length,
4800                         PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
4801                 return -EINVAL;
4802         }
4803
4804         if (ctrl_info->max_inbound_iu_length_per_firmware <
4805                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4806                 dev_err(&ctrl_info->pci_dev->dev,
4807                         "max. inbound IU length of %u is less than the min. required length of %d\n",
4808                         ctrl_info->max_inbound_iu_length_per_firmware,
4809                         PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4810                 return -EINVAL;
4811         }
4812
4813         if (!ctrl_info->inbound_spanning_supported) {
4814                 dev_err(&ctrl_info->pci_dev->dev,
4815                         "the controller does not support inbound spanning\n");
4816                 return -EINVAL;
4817         }
4818
4819         if (ctrl_info->outbound_spanning_supported) {
4820                 dev_err(&ctrl_info->pci_dev->dev,
4821                         "the controller supports outbound spanning but this driver does not\n");
4822                 return -EINVAL;
4823         }
4824
4825         return 0;
4826 }
4827
4828 static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
4829 {
4830         int rc;
4831         struct pqi_event_queue *event_queue;
4832         struct pqi_general_admin_request request;
4833         struct pqi_general_admin_response response;
4834
4835         event_queue = &ctrl_info->event_queue;
4836
4837         /*
4838          * Create OQ (Outbound Queue - device to host queue) to dedicate
4839          * to events.
4840          */
4841         memset(&request, 0, sizeof(request));
4842         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4843         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4844                 &request.header.iu_length);
4845         request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4846         put_unaligned_le16(event_queue->oq_id,
4847                 &request.data.create_operational_oq.queue_id);
4848         put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
4849                 &request.data.create_operational_oq.element_array_addr);
4850         put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
4851                 &request.data.create_operational_oq.pi_addr);
4852         put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
4853                 &request.data.create_operational_oq.num_elements);
4854         put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
4855                 &request.data.create_operational_oq.element_length);
4856         request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4857         put_unaligned_le16(event_queue->int_msg_num,
4858                 &request.data.create_operational_oq.int_msg_num);
4859
4860         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4861                 &response);
4862         if (rc)
4863                 return rc;
4864
4865         event_queue->oq_ci = ctrl_info->iomem_base +
4866                 PQI_DEVICE_REGISTERS_OFFSET +
4867                 get_unaligned_le64(
4868                         &response.data.create_operational_oq.oq_ci_offset);
4869
4870         return 0;
4871 }
4872
4873 static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,
4874         unsigned int group_number)
4875 {
4876         int rc;
4877         struct pqi_queue_group *queue_group;
4878         struct pqi_general_admin_request request;
4879         struct pqi_general_admin_response response;
4880
4881         queue_group = &ctrl_info->queue_groups[group_number];
4882
4883         /*
4884          * Create IQ (Inbound Queue - host to device queue) for
4885          * RAID path.
4886          */
4887         memset(&request, 0, sizeof(request));
4888         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4889         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4890                 &request.header.iu_length);
4891         request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4892         put_unaligned_le16(queue_group->iq_id[RAID_PATH],
4893                 &request.data.create_operational_iq.queue_id);
4894         put_unaligned_le64(
4895                 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
4896                 &request.data.create_operational_iq.element_array_addr);
4897         put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
4898                 &request.data.create_operational_iq.ci_addr);
4899         put_unaligned_le16(ctrl_info->num_elements_per_iq,
4900                 &request.data.create_operational_iq.num_elements);
4901         put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4902                 &request.data.create_operational_iq.element_length);
4903         request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4904
4905         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4906                 &response);
4907         if (rc) {
4908                 dev_err(&ctrl_info->pci_dev->dev,
4909                         "error creating inbound RAID queue\n");
4910                 return rc;
4911         }
4912
4913         queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
4914                 PQI_DEVICE_REGISTERS_OFFSET +
4915                 get_unaligned_le64(
4916                         &response.data.create_operational_iq.iq_pi_offset);
4917
4918         /*
4919          * Create IQ (Inbound Queue - host to device queue) for
4920          * Advanced I/O (AIO) path.
4921          */
4922         memset(&request, 0, sizeof(request));
4923         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4924         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4925                 &request.header.iu_length);
4926         request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4927         put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4928                 &request.data.create_operational_iq.queue_id);
4929         put_unaligned_le64((u64)queue_group->
4930                 iq_element_array_bus_addr[AIO_PATH],
4931                 &request.data.create_operational_iq.element_array_addr);
4932         put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
4933                 &request.data.create_operational_iq.ci_addr);
4934         put_unaligned_le16(ctrl_info->num_elements_per_iq,
4935                 &request.data.create_operational_iq.num_elements);
4936         put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4937                 &request.data.create_operational_iq.element_length);
4938         request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4939
4940         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4941                 &response);
4942         if (rc) {
4943                 dev_err(&ctrl_info->pci_dev->dev,
4944                         "error creating inbound AIO queue\n");
4945                 return rc;
4946         }
4947
4948         queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
4949                 PQI_DEVICE_REGISTERS_OFFSET +
4950                 get_unaligned_le64(
4951                         &response.data.create_operational_iq.iq_pi_offset);
4952
4953         /*
4954          * Designate the 2nd IQ as the AIO path.  By default, all IQs are
4955          * assumed to be for RAID path I/O unless we change the queue's
4956          * property.
4957          */
4958         memset(&request, 0, sizeof(request));
4959         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4960         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4961                 &request.header.iu_length);
4962         request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
4963         put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4964                 &request.data.change_operational_iq_properties.queue_id);
4965         put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
4966                 &request.data.change_operational_iq_properties.vendor_specific);
4967
4968         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4969                 &response);
4970         if (rc) {
4971                 dev_err(&ctrl_info->pci_dev->dev,
4972                         "error changing queue property\n");
4973                 return rc;
4974         }
4975
4976         /*
4977          * Create OQ (Outbound Queue - device to host queue).
4978          */
4979         memset(&request, 0, sizeof(request));
4980         request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4981         put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4982                 &request.header.iu_length);
4983         request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4984         put_unaligned_le16(queue_group->oq_id,
4985                 &request.data.create_operational_oq.queue_id);
4986         put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
4987                 &request.data.create_operational_oq.element_array_addr);
4988         put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
4989                 &request.data.create_operational_oq.pi_addr);
4990         put_unaligned_le16(ctrl_info->num_elements_per_oq,
4991                 &request.data.create_operational_oq.num_elements);
4992         put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
4993                 &request.data.create_operational_oq.element_length);
4994         request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4995         put_unaligned_le16(queue_group->int_msg_num,
4996                 &request.data.create_operational_oq.int_msg_num);
4997
4998         rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4999                 &response);
5000         if (rc) {
5001                 dev_err(&ctrl_info->pci_dev->dev,
5002                         "error creating outbound queue\n");
5003                 return rc;
5004         }
5005
5006         queue_group->oq_ci = ctrl_info->iomem_base +
5007                 PQI_DEVICE_REGISTERS_OFFSET +
5008                 get_unaligned_le64(
5009                         &response.data.create_operational_oq.oq_ci_offset);
5010
5011         return 0;
5012 }
5013
5014 static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
5015 {
5016         int rc;
5017         unsigned int i;
5018
5019         rc = pqi_create_event_queue(ctrl_info);
5020         if (rc) {
5021                 dev_err(&ctrl_info->pci_dev->dev,
5022                         "error creating event queue\n");
5023                 return rc;
5024         }
5025
5026         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5027                 rc = pqi_create_queue_group(ctrl_info, i);
5028                 if (rc) {
5029                         dev_err(&ctrl_info->pci_dev->dev,
5030                                 "error creating queue group number %u/%u\n",
5031                                 i, ctrl_info->num_queue_groups);
5032                         return rc;
5033                 }
5034         }
5035
5036         return 0;
5037 }
5038
5039 #define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH   \
5040         struct_size_t(struct pqi_event_config,  descriptors, PQI_MAX_EVENT_DESCRIPTORS)
5041
5042 static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,
5043         bool enable_events)
5044 {
5045         int rc;
5046         unsigned int i;
5047         struct pqi_event_config *event_config;
5048         struct pqi_event_descriptor *event_descriptor;
5049         struct pqi_general_management_request request;
5050
5051         event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5052                 GFP_KERNEL);
5053         if (!event_config)
5054                 return -ENOMEM;
5055
5056         memset(&request, 0, sizeof(request));
5057
5058         request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
5059         put_unaligned_le16(offsetof(struct pqi_general_management_request,
5060                 data.report_event_configuration.sg_descriptors[1]) -
5061                 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
5062         put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5063                 &request.data.report_event_configuration.buffer_length);
5064
5065         rc = pqi_map_single(ctrl_info->pci_dev,
5066                 request.data.report_event_configuration.sg_descriptors,
5067                 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5068                 DMA_FROM_DEVICE);
5069         if (rc)
5070                 goto out;
5071
5072         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
5073
5074         pqi_pci_unmap(ctrl_info->pci_dev,
5075                 request.data.report_event_configuration.sg_descriptors, 1,
5076                 DMA_FROM_DEVICE);
5077
5078         if (rc)
5079                 goto out;
5080
5081         for (i = 0; i < event_config->num_event_descriptors; i++) {
5082                 event_descriptor = &event_config->descriptors[i];
5083                 if (enable_events &&
5084                         pqi_is_supported_event(event_descriptor->event_type))
5085                                 put_unaligned_le16(ctrl_info->event_queue.oq_id,
5086                                         &event_descriptor->oq_id);
5087                 else
5088                         put_unaligned_le16(0, &event_descriptor->oq_id);
5089         }
5090
5091         memset(&request, 0, sizeof(request));
5092
5093         request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
5094         put_unaligned_le16(offsetof(struct pqi_general_management_request,
5095                 data.report_event_configuration.sg_descriptors[1]) -
5096                 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
5097         put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5098                 &request.data.report_event_configuration.buffer_length);
5099
5100         rc = pqi_map_single(ctrl_info->pci_dev,
5101                 request.data.report_event_configuration.sg_descriptors,
5102                 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5103                 DMA_TO_DEVICE);
5104         if (rc)
5105                 goto out;
5106
5107         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
5108
5109         pqi_pci_unmap(ctrl_info->pci_dev,
5110                 request.data.report_event_configuration.sg_descriptors, 1,
5111                 DMA_TO_DEVICE);
5112
5113 out:
5114         kfree(event_config);
5115
5116         return rc;
5117 }
5118
5119 static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)
5120 {
5121         return pqi_configure_events(ctrl_info, true);
5122 }
5123
5124 static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
5125 {
5126         unsigned int i;
5127         struct device *dev;
5128         size_t sg_chain_buffer_length;
5129         struct pqi_io_request *io_request;
5130
5131         if (!ctrl_info->io_request_pool)
5132                 return;
5133
5134         dev = &ctrl_info->pci_dev->dev;
5135         sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
5136         io_request = ctrl_info->io_request_pool;
5137
5138         for (i = 0; i < ctrl_info->max_io_slots; i++) {
5139                 kfree(io_request->iu);
5140                 if (!io_request->sg_chain_buffer)
5141                         break;
5142                 dma_free_coherent(dev, sg_chain_buffer_length,
5143                         io_request->sg_chain_buffer,
5144                         io_request->sg_chain_buffer_dma_handle);
5145                 io_request++;
5146         }
5147
5148         kfree(ctrl_info->io_request_pool);
5149         ctrl_info->io_request_pool = NULL;
5150 }
5151
5152 static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
5153 {
5154         ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,
5155                                      ctrl_info->error_buffer_length,
5156                                      &ctrl_info->error_buffer_dma_handle,
5157                                      GFP_KERNEL);
5158         if (!ctrl_info->error_buffer)
5159                 return -ENOMEM;
5160
5161         return 0;
5162 }
5163
5164 static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
5165 {
5166         unsigned int i;
5167         void *sg_chain_buffer;
5168         size_t sg_chain_buffer_length;
5169         dma_addr_t sg_chain_buffer_dma_handle;
5170         struct device *dev;
5171         struct pqi_io_request *io_request;
5172
5173         ctrl_info->io_request_pool = kcalloc(ctrl_info->max_io_slots,
5174                 sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);
5175
5176         if (!ctrl_info->io_request_pool) {
5177                 dev_err(&ctrl_info->pci_dev->dev,
5178                         "failed to allocate I/O request pool\n");
5179                 goto error;
5180         }
5181
5182         dev = &ctrl_info->pci_dev->dev;
5183         sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
5184         io_request = ctrl_info->io_request_pool;
5185
5186         for (i = 0; i < ctrl_info->max_io_slots; i++) {
5187                 io_request->iu = kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
5188
5189                 if (!io_request->iu) {
5190                         dev_err(&ctrl_info->pci_dev->dev,
5191                                 "failed to allocate IU buffers\n");
5192                         goto error;
5193                 }
5194
5195                 sg_chain_buffer = dma_alloc_coherent(dev,
5196                         sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
5197                         GFP_KERNEL);
5198
5199                 if (!sg_chain_buffer) {
5200                         dev_err(&ctrl_info->pci_dev->dev,
5201                                 "failed to allocate PQI scatter-gather chain buffers\n");
5202                         goto error;
5203                 }
5204
5205                 io_request->index = i;
5206                 io_request->sg_chain_buffer = sg_chain_buffer;
5207                 io_request->sg_chain_buffer_dma_handle = sg_chain_buffer_dma_handle;
5208                 io_request++;
5209         }
5210
5211         return 0;
5212
5213 error:
5214         pqi_free_all_io_requests(ctrl_info);
5215
5216         return -ENOMEM;
5217 }
5218
5219 /*
5220  * Calculate required resources that are sized based on max. outstanding
5221  * requests and max. transfer size.
5222  */
5223
5224 static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
5225 {
5226         u32 max_transfer_size;
5227         u32 max_sg_entries;
5228
5229         ctrl_info->scsi_ml_can_queue =
5230                 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
5231         ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
5232
5233         ctrl_info->error_buffer_length =
5234                 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
5235
5236         if (reset_devices)
5237                 max_transfer_size = min(ctrl_info->max_transfer_size,
5238                         PQI_MAX_TRANSFER_SIZE_KDUMP);
5239         else
5240                 max_transfer_size = min(ctrl_info->max_transfer_size,
5241                         PQI_MAX_TRANSFER_SIZE);
5242
5243         max_sg_entries = max_transfer_size / PAGE_SIZE;
5244
5245         /* +1 to cover when the buffer is not page-aligned. */
5246         max_sg_entries++;
5247
5248         max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
5249
5250         max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
5251
5252         ctrl_info->sg_chain_buffer_length =
5253                 (max_sg_entries * sizeof(struct pqi_sg_descriptor)) +
5254                 PQI_EXTRA_SGL_MEMORY;
5255         ctrl_info->sg_tablesize = max_sg_entries;
5256         ctrl_info->max_sectors = max_transfer_size / 512;
5257 }
5258
5259 static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
5260 {
5261         int num_queue_groups;
5262         u16 num_elements_per_iq;
5263         u16 num_elements_per_oq;
5264
5265         if (reset_devices) {
5266                 num_queue_groups = 1;
5267         } else {
5268                 int num_cpus;
5269                 int max_queue_groups;
5270
5271                 max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
5272                         ctrl_info->max_outbound_queues - 1);
5273                 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
5274
5275                 num_cpus = num_online_cpus();
5276                 num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);
5277                 num_queue_groups = min(num_queue_groups, max_queue_groups);
5278         }
5279
5280         ctrl_info->num_queue_groups = num_queue_groups;
5281
5282         /*
5283          * Make sure that the max. inbound IU length is an even multiple
5284          * of our inbound element length.
5285          */
5286         ctrl_info->max_inbound_iu_length =
5287                 (ctrl_info->max_inbound_iu_length_per_firmware /
5288                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
5289                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
5290
5291         num_elements_per_iq =
5292                 (ctrl_info->max_inbound_iu_length /
5293                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
5294
5295         /* Add one because one element in each queue is unusable. */
5296         num_elements_per_iq++;
5297
5298         num_elements_per_iq = min(num_elements_per_iq,
5299                 ctrl_info->max_elements_per_iq);
5300
5301         num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
5302         num_elements_per_oq = min(num_elements_per_oq,
5303                 ctrl_info->max_elements_per_oq);
5304
5305         ctrl_info->num_elements_per_iq = num_elements_per_iq;
5306         ctrl_info->num_elements_per_oq = num_elements_per_oq;
5307
5308         ctrl_info->max_sg_per_iu =
5309                 ((ctrl_info->max_inbound_iu_length -
5310                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5311                 sizeof(struct pqi_sg_descriptor)) +
5312                 PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
5313
5314         ctrl_info->max_sg_per_r56_iu =
5315                 ((ctrl_info->max_inbound_iu_length -
5316                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5317                 sizeof(struct pqi_sg_descriptor)) +
5318                 PQI_MAX_EMBEDDED_R56_SG_DESCRIPTORS;
5319 }
5320
5321 static inline void pqi_set_sg_descriptor(struct pqi_sg_descriptor *sg_descriptor,
5322         struct scatterlist *sg)
5323 {
5324         u64 address = (u64)sg_dma_address(sg);
5325         unsigned int length = sg_dma_len(sg);
5326
5327         put_unaligned_le64(address, &sg_descriptor->address);
5328         put_unaligned_le32(length, &sg_descriptor->length);
5329         put_unaligned_le32(0, &sg_descriptor->flags);
5330 }
5331
5332 static unsigned int pqi_build_sg_list(struct pqi_sg_descriptor *sg_descriptor,
5333         struct scatterlist *sg, int sg_count, struct pqi_io_request *io_request,
5334         int max_sg_per_iu, bool *chained)
5335 {
5336         int i;
5337         unsigned int num_sg_in_iu;
5338
5339         *chained = false;
5340         i = 0;
5341         num_sg_in_iu = 0;
5342         max_sg_per_iu--;        /* Subtract 1 to leave room for chain marker. */
5343
5344         while (1) {
5345                 pqi_set_sg_descriptor(sg_descriptor, sg);
5346                 if (!*chained)
5347                         num_sg_in_iu++;
5348                 i++;
5349                 if (i == sg_count)
5350                         break;
5351                 sg_descriptor++;
5352                 if (i == max_sg_per_iu) {
5353                         put_unaligned_le64((u64)io_request->sg_chain_buffer_dma_handle,
5354                                 &sg_descriptor->address);
5355                         put_unaligned_le32((sg_count - num_sg_in_iu) * sizeof(*sg_descriptor),
5356                                 &sg_descriptor->length);
5357                         put_unaligned_le32(CISS_SG_CHAIN, &sg_descriptor->flags);
5358                         *chained = true;
5359                         num_sg_in_iu++;
5360                         sg_descriptor = io_request->sg_chain_buffer;
5361                 }
5362                 sg = sg_next(sg);
5363         }
5364
5365         put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
5366
5367         return num_sg_in_iu;
5368 }
5369
5370 static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
5371         struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
5372         struct pqi_io_request *io_request)
5373 {
5374         u16 iu_length;
5375         int sg_count;
5376         bool chained;
5377         unsigned int num_sg_in_iu;
5378         struct scatterlist *sg;
5379         struct pqi_sg_descriptor *sg_descriptor;
5380
5381         sg_count = scsi_dma_map(scmd);
5382         if (sg_count < 0)
5383                 return sg_count;
5384
5385         iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
5386                 PQI_REQUEST_HEADER_LENGTH;
5387
5388         if (sg_count == 0)
5389                 goto out;
5390
5391         sg = scsi_sglist(scmd);
5392         sg_descriptor = request->sg_descriptors;
5393
5394         num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5395                 ctrl_info->max_sg_per_iu, &chained);
5396
5397         request->partial = chained;
5398         iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5399
5400 out:
5401         put_unaligned_le16(iu_length, &request->header.iu_length);
5402
5403         return 0;
5404 }
5405
5406 static int pqi_build_aio_r1_sg_list(struct pqi_ctrl_info *ctrl_info,
5407         struct pqi_aio_r1_path_request *request, struct scsi_cmnd *scmd,
5408         struct pqi_io_request *io_request)
5409 {
5410         u16 iu_length;
5411         int sg_count;
5412         bool chained;
5413         unsigned int num_sg_in_iu;
5414         struct scatterlist *sg;
5415         struct pqi_sg_descriptor *sg_descriptor;
5416
5417         sg_count = scsi_dma_map(scmd);
5418         if (sg_count < 0)
5419                 return sg_count;
5420
5421         iu_length = offsetof(struct pqi_aio_r1_path_request, sg_descriptors) -
5422                 PQI_REQUEST_HEADER_LENGTH;
5423         num_sg_in_iu = 0;
5424
5425         if (sg_count == 0)
5426                 goto out;
5427
5428         sg = scsi_sglist(scmd);
5429         sg_descriptor = request->sg_descriptors;
5430
5431         num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5432                 ctrl_info->max_sg_per_iu, &chained);
5433
5434         request->partial = chained;
5435         iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5436
5437 out:
5438         put_unaligned_le16(iu_length, &request->header.iu_length);
5439         request->num_sg_descriptors = num_sg_in_iu;
5440
5441         return 0;
5442 }
5443
5444 static int pqi_build_aio_r56_sg_list(struct pqi_ctrl_info *ctrl_info,
5445         struct pqi_aio_r56_path_request *request, struct scsi_cmnd *scmd,
5446         struct pqi_io_request *io_request)
5447 {
5448         u16 iu_length;
5449         int sg_count;
5450         bool chained;
5451         unsigned int num_sg_in_iu;
5452         struct scatterlist *sg;
5453         struct pqi_sg_descriptor *sg_descriptor;
5454
5455         sg_count = scsi_dma_map(scmd);
5456         if (sg_count < 0)
5457                 return sg_count;
5458
5459         iu_length = offsetof(struct pqi_aio_r56_path_request, sg_descriptors) -
5460                 PQI_REQUEST_HEADER_LENGTH;
5461         num_sg_in_iu = 0;
5462
5463         if (sg_count != 0) {
5464                 sg = scsi_sglist(scmd);
5465                 sg_descriptor = request->sg_descriptors;
5466
5467                 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5468                         ctrl_info->max_sg_per_r56_iu, &chained);
5469
5470                 request->partial = chained;
5471                 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5472         }
5473
5474         put_unaligned_le16(iu_length, &request->header.iu_length);
5475         request->num_sg_descriptors = num_sg_in_iu;
5476
5477         return 0;
5478 }
5479
5480 static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
5481         struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
5482         struct pqi_io_request *io_request)
5483 {
5484         u16 iu_length;
5485         int sg_count;
5486         bool chained;
5487         unsigned int num_sg_in_iu;
5488         struct scatterlist *sg;
5489         struct pqi_sg_descriptor *sg_descriptor;
5490
5491         sg_count = scsi_dma_map(scmd);
5492         if (sg_count < 0)
5493                 return sg_count;
5494
5495         iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
5496                 PQI_REQUEST_HEADER_LENGTH;
5497         num_sg_in_iu = 0;
5498
5499         if (sg_count == 0)
5500                 goto out;
5501
5502         sg = scsi_sglist(scmd);
5503         sg_descriptor = request->sg_descriptors;
5504
5505         num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5506                 ctrl_info->max_sg_per_iu, &chained);
5507
5508         request->partial = chained;
5509         iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5510
5511 out:
5512         put_unaligned_le16(iu_length, &request->header.iu_length);
5513         request->num_sg_descriptors = num_sg_in_iu;
5514
5515         return 0;
5516 }
5517
5518 static void pqi_raid_io_complete(struct pqi_io_request *io_request,
5519         void *context)
5520 {
5521         struct scsi_cmnd *scmd;
5522
5523         scmd = io_request->scmd;
5524         pqi_free_io_request(io_request);
5525         scsi_dma_unmap(scmd);
5526         pqi_scsi_done(scmd);
5527 }
5528
5529 static int pqi_raid_submit_io(struct pqi_ctrl_info *ctrl_info,
5530         struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5531         struct pqi_queue_group *queue_group, bool io_high_prio)
5532 {
5533         int rc;
5534         size_t cdb_length;
5535         struct pqi_io_request *io_request;
5536         struct pqi_raid_path_request *request;
5537
5538         io_request = pqi_alloc_io_request(ctrl_info, scmd);
5539         if (!io_request)
5540                 return SCSI_MLQUEUE_HOST_BUSY;
5541
5542         io_request->io_complete_callback = pqi_raid_io_complete;
5543         io_request->scmd = scmd;
5544
5545         request = io_request->iu;
5546         memset(request, 0, offsetof(struct pqi_raid_path_request, sg_descriptors));
5547
5548         request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
5549         put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5550         request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5551         request->command_priority = io_high_prio;
5552         put_unaligned_le16(io_request->index, &request->request_id);
5553         request->error_index = request->request_id;
5554         memcpy(request->lun_number, device->scsi3addr, sizeof(request->lun_number));
5555         request->ml_device_lun_number = (u8)scmd->device->lun;
5556
5557         cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
5558         memcpy(request->cdb, scmd->cmnd, cdb_length);
5559
5560         switch (cdb_length) {
5561         case 6:
5562         case 10:
5563         case 12:
5564         case 16:
5565                 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
5566                 break;
5567         case 20:
5568                 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_4;
5569                 break;
5570         case 24:
5571                 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_8;
5572                 break;
5573         case 28:
5574                 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_12;
5575                 break;
5576         case 32:
5577         default:
5578                 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_16;
5579                 break;
5580         }
5581
5582         switch (scmd->sc_data_direction) {
5583         case DMA_FROM_DEVICE:
5584                 request->data_direction = SOP_READ_FLAG;
5585                 break;
5586         case DMA_TO_DEVICE:
5587                 request->data_direction = SOP_WRITE_FLAG;
5588                 break;
5589         case DMA_NONE:
5590                 request->data_direction = SOP_NO_DIRECTION_FLAG;
5591                 break;
5592         case DMA_BIDIRECTIONAL:
5593                 request->data_direction = SOP_BIDIRECTIONAL;
5594                 break;
5595         default:
5596                 dev_err(&ctrl_info->pci_dev->dev,
5597                         "unknown data direction: %d\n",
5598                         scmd->sc_data_direction);
5599                 break;
5600         }
5601
5602         rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
5603         if (rc) {
5604                 pqi_free_io_request(io_request);
5605                 return SCSI_MLQUEUE_HOST_BUSY;
5606         }
5607
5608         pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
5609
5610         return 0;
5611 }
5612
5613 static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5614         struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5615         struct pqi_queue_group *queue_group)
5616 {
5617         bool io_high_prio;
5618
5619         io_high_prio = pqi_is_io_high_priority(device, scmd);
5620
5621         return pqi_raid_submit_io(ctrl_info, device, scmd, queue_group, io_high_prio);
5622 }
5623
5624 static bool pqi_raid_bypass_retry_needed(struct pqi_io_request *io_request)
5625 {
5626         struct scsi_cmnd *scmd;
5627         struct pqi_scsi_dev *device;
5628         struct pqi_ctrl_info *ctrl_info;
5629
5630         if (!io_request->raid_bypass)
5631                 return false;
5632
5633         scmd = io_request->scmd;
5634         if ((scmd->result & 0xff) == SAM_STAT_GOOD)
5635                 return false;
5636         if (host_byte(scmd->result) == DID_NO_CONNECT)
5637                 return false;
5638
5639         device = scmd->device->hostdata;
5640         if (pqi_device_offline(device) || pqi_device_in_remove(device))
5641                 return false;
5642
5643         ctrl_info = shost_to_hba(scmd->device->host);
5644         if (pqi_ctrl_offline(ctrl_info))
5645                 return false;
5646
5647         return true;
5648 }
5649
5650 static void pqi_aio_io_complete(struct pqi_io_request *io_request,
5651         void *context)
5652 {
5653         struct scsi_cmnd *scmd;
5654
5655         scmd = io_request->scmd;
5656         scsi_dma_unmap(scmd);
5657         if (io_request->status == -EAGAIN || pqi_raid_bypass_retry_needed(io_request)) {
5658                 set_host_byte(scmd, DID_IMM_RETRY);
5659                 pqi_cmd_priv(scmd)->this_residual++;
5660         }
5661
5662         pqi_free_io_request(io_request);
5663         pqi_scsi_done(scmd);
5664 }
5665
5666 static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5667         struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5668         struct pqi_queue_group *queue_group)
5669 {
5670         bool io_high_prio;
5671
5672         io_high_prio = pqi_is_io_high_priority(device, scmd);
5673
5674         return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
5675                 scmd->cmnd, scmd->cmd_len, queue_group, NULL,
5676                 false, io_high_prio);
5677 }
5678
5679 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
5680         struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
5681         unsigned int cdb_length, struct pqi_queue_group *queue_group,
5682         struct pqi_encryption_info *encryption_info, bool raid_bypass,
5683         bool io_high_prio)
5684 {
5685         int rc;
5686         struct pqi_io_request *io_request;
5687         struct pqi_aio_path_request *request;
5688
5689         io_request = pqi_alloc_io_request(ctrl_info, scmd);
5690         if (!io_request)
5691                 return SCSI_MLQUEUE_HOST_BUSY;
5692
5693         io_request->io_complete_callback = pqi_aio_io_complete;
5694         io_request->scmd = scmd;
5695         io_request->raid_bypass = raid_bypass;
5696
5697         request = io_request->iu;
5698         memset(request, 0, offsetof(struct pqi_aio_path_request, sg_descriptors));
5699
5700         request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
5701         put_unaligned_le32(aio_handle, &request->nexus_id);
5702         put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5703         request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5704         request->command_priority = io_high_prio;
5705         put_unaligned_le16(io_request->index, &request->request_id);
5706         request->error_index = request->request_id;
5707         if (!raid_bypass && ctrl_info->multi_lun_device_supported)
5708                 put_unaligned_le64(scmd->device->lun << 8, &request->lun_number);
5709         if (cdb_length > sizeof(request->cdb))
5710                 cdb_length = sizeof(request->cdb);
5711         request->cdb_length = cdb_length;
5712         memcpy(request->cdb, cdb, cdb_length);
5713
5714         switch (scmd->sc_data_direction) {
5715         case DMA_TO_DEVICE:
5716                 request->data_direction = SOP_READ_FLAG;
5717                 break;
5718         case DMA_FROM_DEVICE:
5719                 request->data_direction = SOP_WRITE_FLAG;
5720                 break;
5721         case DMA_NONE:
5722                 request->data_direction = SOP_NO_DIRECTION_FLAG;
5723                 break;
5724         case DMA_BIDIRECTIONAL:
5725                 request->data_direction = SOP_BIDIRECTIONAL;
5726                 break;
5727         default:
5728                 dev_err(&ctrl_info->pci_dev->dev,
5729                         "unknown data direction: %d\n",
5730                         scmd->sc_data_direction);
5731                 break;
5732         }
5733
5734         if (encryption_info) {
5735                 request->encryption_enable = true;
5736                 put_unaligned_le16(encryption_info->data_encryption_key_index,
5737                         &request->data_encryption_key_index);
5738                 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5739                         &request->encrypt_tweak_lower);
5740                 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5741                         &request->encrypt_tweak_upper);
5742         }
5743
5744         rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
5745         if (rc) {
5746                 pqi_free_io_request(io_request);
5747                 return SCSI_MLQUEUE_HOST_BUSY;
5748         }
5749
5750         pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5751
5752         return 0;
5753 }
5754
5755 static  int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
5756         struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5757         struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5758         struct pqi_scsi_dev_raid_map_data *rmd)
5759 {
5760         int rc;
5761         struct pqi_io_request *io_request;
5762         struct pqi_aio_r1_path_request *r1_request;
5763
5764         io_request = pqi_alloc_io_request(ctrl_info, scmd);
5765         if (!io_request)
5766                 return SCSI_MLQUEUE_HOST_BUSY;
5767
5768         io_request->io_complete_callback = pqi_aio_io_complete;
5769         io_request->scmd = scmd;
5770         io_request->raid_bypass = true;
5771
5772         r1_request = io_request->iu;
5773         memset(r1_request, 0, offsetof(struct pqi_aio_r1_path_request, sg_descriptors));
5774
5775         r1_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID1_IO;
5776         put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r1_request->volume_id);
5777         r1_request->num_drives = rmd->num_it_nexus_entries;
5778         put_unaligned_le32(rmd->it_nexus[0], &r1_request->it_nexus_1);
5779         put_unaligned_le32(rmd->it_nexus[1], &r1_request->it_nexus_2);
5780         if (rmd->num_it_nexus_entries == 3)
5781                 put_unaligned_le32(rmd->it_nexus[2], &r1_request->it_nexus_3);
5782
5783         put_unaligned_le32(scsi_bufflen(scmd), &r1_request->data_length);
5784         r1_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5785         put_unaligned_le16(io_request->index, &r1_request->request_id);
5786         r1_request->error_index = r1_request->request_id;
5787         if (rmd->cdb_length > sizeof(r1_request->cdb))
5788                 rmd->cdb_length = sizeof(r1_request->cdb);
5789         r1_request->cdb_length = rmd->cdb_length;
5790         memcpy(r1_request->cdb, rmd->cdb, rmd->cdb_length);
5791
5792         /* The direction is always write. */
5793         r1_request->data_direction = SOP_READ_FLAG;
5794
5795         if (encryption_info) {
5796                 r1_request->encryption_enable = true;
5797                 put_unaligned_le16(encryption_info->data_encryption_key_index,
5798                                 &r1_request->data_encryption_key_index);
5799                 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5800                                 &r1_request->encrypt_tweak_lower);
5801                 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5802                                 &r1_request->encrypt_tweak_upper);
5803         }
5804
5805         rc = pqi_build_aio_r1_sg_list(ctrl_info, r1_request, scmd, io_request);
5806         if (rc) {
5807                 pqi_free_io_request(io_request);
5808                 return SCSI_MLQUEUE_HOST_BUSY;
5809         }
5810
5811         pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5812
5813         return 0;
5814 }
5815
5816 static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
5817         struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5818         struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5819         struct pqi_scsi_dev_raid_map_data *rmd)
5820 {
5821         int rc;
5822         struct pqi_io_request *io_request;
5823         struct pqi_aio_r56_path_request *r56_request;
5824
5825         io_request = pqi_alloc_io_request(ctrl_info, scmd);
5826         if (!io_request)
5827                 return SCSI_MLQUEUE_HOST_BUSY;
5828         io_request->io_complete_callback = pqi_aio_io_complete;
5829         io_request->scmd = scmd;
5830         io_request->raid_bypass = true;
5831
5832         r56_request = io_request->iu;
5833         memset(r56_request, 0, offsetof(struct pqi_aio_r56_path_request, sg_descriptors));
5834
5835         if (device->raid_level == SA_RAID_5 || device->raid_level == SA_RAID_51)
5836                 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID5_IO;
5837         else
5838                 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID6_IO;
5839
5840         put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r56_request->volume_id);
5841         put_unaligned_le32(rmd->aio_handle, &r56_request->data_it_nexus);
5842         put_unaligned_le32(rmd->p_parity_it_nexus, &r56_request->p_parity_it_nexus);
5843         if (rmd->raid_level == SA_RAID_6) {
5844                 put_unaligned_le32(rmd->q_parity_it_nexus, &r56_request->q_parity_it_nexus);
5845                 r56_request->xor_multiplier = rmd->xor_mult;
5846         }
5847         put_unaligned_le32(scsi_bufflen(scmd), &r56_request->data_length);
5848         r56_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5849         put_unaligned_le64(rmd->row, &r56_request->row);
5850
5851         put_unaligned_le16(io_request->index, &r56_request->request_id);
5852         r56_request->error_index = r56_request->request_id;
5853
5854         if (rmd->cdb_length > sizeof(r56_request->cdb))
5855                 rmd->cdb_length = sizeof(r56_request->cdb);
5856         r56_request->cdb_length = rmd->cdb_length;
5857         memcpy(r56_request->cdb, rmd->cdb, rmd->cdb_length);
5858
5859         /* The direction is always write. */
5860         r56_request->data_direction = SOP_READ_FLAG;
5861
5862         if (encryption_info) {
5863                 r56_request->encryption_enable = true;
5864                 put_unaligned_le16(encryption_info->data_encryption_key_index,
5865                                 &r56_request->data_encryption_key_index);
5866                 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5867                                 &r56_request->encrypt_tweak_lower);
5868                 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5869                                 &r56_request->encrypt_tweak_upper);
5870         }
5871
5872         rc = pqi_build_aio_r56_sg_list(ctrl_info, r56_request, scmd, io_request);
5873         if (rc) {
5874                 pqi_free_io_request(io_request);
5875                 return SCSI_MLQUEUE_HOST_BUSY;
5876         }
5877
5878         pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5879
5880         return 0;
5881 }
5882
5883 static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,
5884         struct scsi_cmnd *scmd)
5885 {
5886         /*
5887          * We are setting host_tagset = 1 during init.
5888          */
5889         return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scsi_cmd_to_rq(scmd)));
5890 }
5891
5892 static inline bool pqi_is_bypass_eligible_request(struct scsi_cmnd *scmd)
5893 {
5894         if (blk_rq_is_passthrough(scsi_cmd_to_rq(scmd)))
5895                 return false;
5896
5897         return pqi_cmd_priv(scmd)->this_residual == 0;
5898 }
5899
5900 /*
5901  * This function gets called just before we hand the completed SCSI request
5902  * back to the SML.
5903  */
5904
5905 void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd)
5906 {
5907         struct pqi_scsi_dev *device;
5908         struct completion *wait;
5909
5910         if (!scmd->device) {
5911                 set_host_byte(scmd, DID_NO_CONNECT);
5912                 return;
5913         }
5914
5915         device = scmd->device->hostdata;
5916         if (!device) {
5917                 set_host_byte(scmd, DID_NO_CONNECT);
5918                 return;
5919         }
5920
5921         atomic_dec(&device->scsi_cmds_outstanding[scmd->device->lun]);
5922
5923         wait = (struct completion *)xchg(&scmd->host_scribble, NULL);
5924         if (wait != PQI_NO_COMPLETION)
5925                 complete(wait);
5926 }
5927
5928 static bool pqi_is_parity_write_stream(struct pqi_ctrl_info *ctrl_info,
5929         struct scsi_cmnd *scmd)
5930 {
5931         u32 oldest_jiffies;
5932         u8 lru_index;
5933         int i;
5934         int rc;
5935         struct pqi_scsi_dev *device;
5936         struct pqi_stream_data *pqi_stream_data;
5937         struct pqi_scsi_dev_raid_map_data rmd;
5938
5939         if (!ctrl_info->enable_stream_detection)
5940                 return false;
5941
5942         rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
5943         if (rc)
5944                 return false;
5945
5946         /* Check writes only. */
5947         if (!rmd.is_write)
5948                 return false;
5949
5950         device = scmd->device->hostdata;
5951
5952         /* Check for RAID 5/6 streams. */
5953         if (device->raid_level != SA_RAID_5 && device->raid_level != SA_RAID_6)
5954                 return false;
5955
5956         /*
5957          * If controller does not support AIO RAID{5,6} writes, need to send
5958          * requests down non-AIO path.
5959          */
5960         if ((device->raid_level == SA_RAID_5 && !ctrl_info->enable_r5_writes) ||
5961                 (device->raid_level == SA_RAID_6 && !ctrl_info->enable_r6_writes))
5962                 return true;
5963
5964         lru_index = 0;
5965         oldest_jiffies = INT_MAX;
5966         for (i = 0; i < NUM_STREAMS_PER_LUN; i++) {
5967                 pqi_stream_data = &device->stream_data[i];
5968                 /*
5969                  * Check for adjacent request or request is within
5970                  * the previous request.
5971                  */
5972                 if ((pqi_stream_data->next_lba &&
5973                         rmd.first_block >= pqi_stream_data->next_lba) &&
5974                         rmd.first_block <= pqi_stream_data->next_lba +
5975                                 rmd.block_cnt) {
5976                         pqi_stream_data->next_lba = rmd.first_block +
5977                                 rmd.block_cnt;
5978                         pqi_stream_data->last_accessed = jiffies;
5979                         return true;
5980                 }
5981
5982                 /* unused entry */
5983                 if (pqi_stream_data->last_accessed == 0) {
5984                         lru_index = i;
5985                         break;
5986                 }
5987
5988                 /* Find entry with oldest last accessed time. */
5989                 if (pqi_stream_data->last_accessed <= oldest_jiffies) {
5990                         oldest_jiffies = pqi_stream_data->last_accessed;
5991                         lru_index = i;
5992                 }
5993         }
5994
5995         /* Set LRU entry. */
5996         pqi_stream_data = &device->stream_data[lru_index];
5997         pqi_stream_data->last_accessed = jiffies;
5998         pqi_stream_data->next_lba = rmd.first_block + rmd.block_cnt;
5999
6000         return false;
6001 }
6002
6003 static int pqi_scsi_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
6004 {
6005         int rc;
6006         struct pqi_ctrl_info *ctrl_info;
6007         struct pqi_scsi_dev *device;
6008         u16 hw_queue;
6009         struct pqi_queue_group *queue_group;
6010         bool raid_bypassed;
6011         u8 lun;
6012
6013         scmd->host_scribble = PQI_NO_COMPLETION;
6014
6015         device = scmd->device->hostdata;
6016
6017         if (!device) {
6018                 set_host_byte(scmd, DID_NO_CONNECT);
6019                 pqi_scsi_done(scmd);
6020                 return 0;
6021         }
6022
6023         lun = (u8)scmd->device->lun;
6024
6025         atomic_inc(&device->scsi_cmds_outstanding[lun]);
6026
6027         ctrl_info = shost_to_hba(shost);
6028
6029         if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(device)) {
6030                 set_host_byte(scmd, DID_NO_CONNECT);
6031                 pqi_scsi_done(scmd);
6032                 return 0;
6033         }
6034
6035         if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device, lun)) {
6036                 rc = SCSI_MLQUEUE_HOST_BUSY;
6037                 goto out;
6038         }
6039
6040         /*
6041          * This is necessary because the SML doesn't zero out this field during
6042          * error recovery.
6043          */
6044         scmd->result = 0;
6045
6046         hw_queue = pqi_get_hw_queue(ctrl_info, scmd);
6047         queue_group = &ctrl_info->queue_groups[hw_queue];
6048
6049         if (pqi_is_logical_device(device)) {
6050                 raid_bypassed = false;
6051                 if (device->raid_bypass_enabled &&
6052                         pqi_is_bypass_eligible_request(scmd) &&
6053                         !pqi_is_parity_write_stream(ctrl_info, scmd)) {
6054                         rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6055                         if (rc == 0 || rc == SCSI_MLQUEUE_HOST_BUSY) {
6056                                 raid_bypassed = true;
6057                                 device->raid_bypass_cnt++;
6058                         }
6059                 }
6060                 if (!raid_bypassed)
6061                         rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6062         } else {
6063                 if (device->aio_enabled)
6064                         rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6065                 else
6066                         rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6067         }
6068
6069 out:
6070         if (rc) {
6071                 scmd->host_scribble = NULL;
6072                 atomic_dec(&device->scsi_cmds_outstanding[lun]);
6073         }
6074
6075         return rc;
6076 }
6077
6078 static unsigned int pqi_queued_io_count(struct pqi_ctrl_info *ctrl_info)
6079 {
6080         unsigned int i;
6081         unsigned int path;
6082         unsigned long flags;
6083         unsigned int queued_io_count;
6084         struct pqi_queue_group *queue_group;
6085         struct pqi_io_request *io_request;
6086
6087         queued_io_count = 0;
6088
6089         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6090                 queue_group = &ctrl_info->queue_groups[i];
6091                 for (path = 0; path < 2; path++) {
6092                         spin_lock_irqsave(&queue_group->submit_lock[path], flags);
6093                         list_for_each_entry(io_request, &queue_group->request_list[path], request_list_entry)
6094                                 queued_io_count++;
6095                         spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
6096                 }
6097         }
6098
6099         return queued_io_count;
6100 }
6101
6102 static unsigned int pqi_nonempty_inbound_queue_count(struct pqi_ctrl_info *ctrl_info)
6103 {
6104         unsigned int i;
6105         unsigned int path;
6106         unsigned int nonempty_inbound_queue_count;
6107         struct pqi_queue_group *queue_group;
6108         pqi_index_t iq_pi;
6109         pqi_index_t iq_ci;
6110
6111         nonempty_inbound_queue_count = 0;
6112
6113         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6114                 queue_group = &ctrl_info->queue_groups[i];
6115                 for (path = 0; path < 2; path++) {
6116                         iq_pi = queue_group->iq_pi_copy[path];
6117                         iq_ci = readl(queue_group->iq_ci[path]);
6118                         if (iq_ci != iq_pi)
6119                                 nonempty_inbound_queue_count++;
6120                 }
6121         }
6122
6123         return nonempty_inbound_queue_count;
6124 }
6125
6126 #define PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS        10
6127
6128 static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)
6129 {
6130         unsigned long start_jiffies;
6131         unsigned long warning_timeout;
6132         unsigned int queued_io_count;
6133         unsigned int nonempty_inbound_queue_count;
6134         bool displayed_warning;
6135
6136         displayed_warning = false;
6137         start_jiffies = jiffies;
6138         warning_timeout = (PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
6139
6140         while (1) {
6141                 queued_io_count = pqi_queued_io_count(ctrl_info);
6142                 nonempty_inbound_queue_count = pqi_nonempty_inbound_queue_count(ctrl_info);
6143                 if (queued_io_count == 0 && nonempty_inbound_queue_count == 0)
6144                         break;
6145                 pqi_check_ctrl_health(ctrl_info);
6146                 if (pqi_ctrl_offline(ctrl_info))
6147                         return -ENXIO;
6148                 if (time_after(jiffies, warning_timeout)) {
6149                         dev_warn(&ctrl_info->pci_dev->dev,
6150                                 "waiting %u seconds for queued I/O to drain (queued I/O count: %u; non-empty inbound queue count: %u)\n",
6151                                 jiffies_to_msecs(jiffies - start_jiffies) / 1000, queued_io_count, nonempty_inbound_queue_count);
6152                         displayed_warning = true;
6153                         warning_timeout = (PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS * HZ) + jiffies;
6154                 }
6155                 usleep_range(1000, 2000);
6156         }
6157
6158         if (displayed_warning)
6159                 dev_warn(&ctrl_info->pci_dev->dev,
6160                         "queued I/O drained after waiting for %u seconds\n",
6161                         jiffies_to_msecs(jiffies - start_jiffies) / 1000);
6162
6163         return 0;
6164 }
6165
6166 static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,
6167         struct pqi_scsi_dev *device, u8 lun)
6168 {
6169         unsigned int i;
6170         unsigned int path;
6171         struct pqi_queue_group *queue_group;
6172         unsigned long flags;
6173         struct pqi_io_request *io_request;
6174         struct pqi_io_request *next;
6175         struct scsi_cmnd *scmd;
6176         struct pqi_scsi_dev *scsi_device;
6177
6178         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6179                 queue_group = &ctrl_info->queue_groups[i];
6180
6181                 for (path = 0; path < 2; path++) {
6182                         spin_lock_irqsave(
6183                                 &queue_group->submit_lock[path], flags);
6184
6185                         list_for_each_entry_safe(io_request, next,
6186                                 &queue_group->request_list[path],
6187                                 request_list_entry) {
6188
6189                                 scmd = io_request->scmd;
6190                                 if (!scmd)
6191                                         continue;
6192
6193                                 scsi_device = scmd->device->hostdata;
6194                                 if (scsi_device != device)
6195                                         continue;
6196
6197                                 if ((u8)scmd->device->lun != lun)
6198                                         continue;
6199
6200                                 list_del(&io_request->request_list_entry);
6201                                 set_host_byte(scmd, DID_RESET);
6202                                 pqi_free_io_request(io_request);
6203                                 scsi_dma_unmap(scmd);
6204                                 pqi_scsi_done(scmd);
6205                         }
6206
6207                         spin_unlock_irqrestore(
6208                                 &queue_group->submit_lock[path], flags);
6209                 }
6210         }
6211 }
6212
6213 #define PQI_PENDING_IO_WARNING_TIMEOUT_SECS     10
6214
6215 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
6216         struct pqi_scsi_dev *device, u8 lun, unsigned long timeout_msecs)
6217 {
6218         int cmds_outstanding;
6219         unsigned long start_jiffies;
6220         unsigned long warning_timeout;
6221         unsigned long msecs_waiting;
6222
6223         start_jiffies = jiffies;
6224         warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
6225
6226         while ((cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun])) > 0) {
6227                 if (ctrl_info->ctrl_removal_state != PQI_CTRL_GRACEFUL_REMOVAL) {
6228                         pqi_check_ctrl_health(ctrl_info);
6229                         if (pqi_ctrl_offline(ctrl_info))
6230                                 return -ENXIO;
6231                 }
6232                 msecs_waiting = jiffies_to_msecs(jiffies - start_jiffies);
6233                 if (msecs_waiting >= timeout_msecs) {
6234                         dev_err(&ctrl_info->pci_dev->dev,
6235                                 "scsi %d:%d:%d:%d: timed out after %lu seconds waiting for %d outstanding command(s)\n",
6236                                 ctrl_info->scsi_host->host_no, device->bus, device->target,
6237                                 lun, msecs_waiting / 1000, cmds_outstanding);
6238                         return -ETIMEDOUT;
6239                 }
6240                 if (time_after(jiffies, warning_timeout)) {
6241                         dev_warn(&ctrl_info->pci_dev->dev,
6242                                 "scsi %d:%d:%d:%d: waiting %lu seconds for %d outstanding command(s)\n",
6243                                 ctrl_info->scsi_host->host_no, device->bus, device->target,
6244                                 lun, msecs_waiting / 1000, cmds_outstanding);
6245                         warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * HZ) + jiffies;
6246                 }
6247                 usleep_range(1000, 2000);
6248         }
6249
6250         return 0;
6251 }
6252
6253 static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
6254         void *context)
6255 {
6256         struct completion *waiting = context;
6257
6258         complete(waiting);
6259 }
6260
6261 #define PQI_LUN_RESET_POLL_COMPLETION_SECS      10
6262
6263 static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
6264         struct pqi_scsi_dev *device, u8 lun, struct completion *wait)
6265 {
6266         int rc;
6267         unsigned int wait_secs;
6268         int cmds_outstanding;
6269
6270         wait_secs = 0;
6271
6272         while (1) {
6273                 if (wait_for_completion_io_timeout(wait,
6274                         PQI_LUN_RESET_POLL_COMPLETION_SECS * HZ)) {
6275                         rc = 0;
6276                         break;
6277                 }
6278
6279                 pqi_check_ctrl_health(ctrl_info);
6280                 if (pqi_ctrl_offline(ctrl_info)) {
6281                         rc = -ENXIO;
6282                         break;
6283                 }
6284
6285                 wait_secs += PQI_LUN_RESET_POLL_COMPLETION_SECS;
6286                 cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun]);
6287                 dev_warn(&ctrl_info->pci_dev->dev,
6288                         "scsi %d:%d:%d:%d: waiting %u seconds for LUN reset to complete (%d command(s) outstanding)\n",
6289                         ctrl_info->scsi_host->host_no, device->bus, device->target, lun, wait_secs, cmds_outstanding);
6290         }
6291
6292         return rc;
6293 }
6294
6295 #define PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS     30
6296
6297 static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6298 {
6299         int rc;
6300         struct pqi_io_request *io_request;
6301         DECLARE_COMPLETION_ONSTACK(wait);
6302         struct pqi_task_management_request *request;
6303
6304         io_request = pqi_alloc_io_request(ctrl_info, NULL);
6305         io_request->io_complete_callback = pqi_lun_reset_complete;
6306         io_request->context = &wait;
6307
6308         request = io_request->iu;
6309         memset(request, 0, sizeof(*request));
6310
6311         request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
6312         put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
6313                 &request->header.iu_length);
6314         put_unaligned_le16(io_request->index, &request->request_id);
6315         memcpy(request->lun_number, device->scsi3addr,
6316                 sizeof(request->lun_number));
6317         if (!pqi_is_logical_device(device) && ctrl_info->multi_lun_device_supported)
6318                 request->ml_device_lun_number = lun;
6319         request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
6320         if (ctrl_info->tmf_iu_timeout_supported)
6321                 put_unaligned_le16(PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS, &request->timeout);
6322
6323         pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
6324                 io_request);
6325
6326         rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, lun, &wait);
6327         if (rc == 0)
6328                 rc = io_request->status;
6329
6330         pqi_free_io_request(io_request);
6331
6332         return rc;
6333 }
6334
6335 #define PQI_LUN_RESET_RETRIES                           3
6336 #define PQI_LUN_RESET_RETRY_INTERVAL_MSECS              (10 * 1000)
6337 #define PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS          (10 * 60 * 1000)
6338 #define PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS   (2 * 60 * 1000)
6339
6340 static int pqi_lun_reset_with_retries(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6341 {
6342         int reset_rc;
6343         int wait_rc;
6344         unsigned int retries;
6345         unsigned long timeout_msecs;
6346
6347         for (retries = 0;;) {
6348                 reset_rc = pqi_lun_reset(ctrl_info, device, lun);
6349                 if (reset_rc == 0 || reset_rc == -ENODEV || reset_rc == -ENXIO || ++retries > PQI_LUN_RESET_RETRIES)
6350                         break;
6351                 msleep(PQI_LUN_RESET_RETRY_INTERVAL_MSECS);
6352         }
6353
6354         timeout_msecs = reset_rc ? PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS :
6355                 PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS;
6356
6357         wait_rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun, timeout_msecs);
6358         if (wait_rc && reset_rc == 0)
6359                 reset_rc = wait_rc;
6360
6361         return reset_rc == 0 ? SUCCESS : FAILED;
6362 }
6363
6364 static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6365 {
6366         int rc;
6367
6368         pqi_ctrl_block_requests(ctrl_info);
6369         pqi_ctrl_wait_until_quiesced(ctrl_info);
6370         pqi_fail_io_queued_for_device(ctrl_info, device, lun);
6371         rc = pqi_wait_until_inbound_queues_empty(ctrl_info);
6372         pqi_device_reset_start(device, lun);
6373         pqi_ctrl_unblock_requests(ctrl_info);
6374         if (rc)
6375                 rc = FAILED;
6376         else
6377                 rc = pqi_lun_reset_with_retries(ctrl_info, device, lun);
6378         pqi_device_reset_done(device, lun);
6379
6380         return rc;
6381 }
6382
6383 static int pqi_device_reset_handler(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun, struct scsi_cmnd *scmd, u8 scsi_opcode)
6384 {
6385         int rc;
6386
6387         mutex_lock(&ctrl_info->lun_reset_mutex);
6388
6389         dev_err(&ctrl_info->pci_dev->dev,
6390                 "resetting scsi %d:%d:%d:%u SCSI cmd at %p due to cmd opcode 0x%02x\n",
6391                 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, scmd, scsi_opcode);
6392
6393         pqi_check_ctrl_health(ctrl_info);
6394         if (pqi_ctrl_offline(ctrl_info))
6395                 rc = FAILED;
6396         else
6397                 rc = pqi_device_reset(ctrl_info, device, lun);
6398
6399         dev_err(&ctrl_info->pci_dev->dev,
6400                 "reset of scsi %d:%d:%d:%u: %s\n",
6401                 ctrl_info->scsi_host->host_no, device->bus, device->target, lun,
6402                 rc == SUCCESS ? "SUCCESS" : "FAILED");
6403
6404         mutex_unlock(&ctrl_info->lun_reset_mutex);
6405
6406         return rc;
6407 }
6408
6409 static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
6410 {
6411         struct Scsi_Host *shost;
6412         struct pqi_ctrl_info *ctrl_info;
6413         struct pqi_scsi_dev *device;
6414         u8 scsi_opcode;
6415
6416         shost = scmd->device->host;
6417         ctrl_info = shost_to_hba(shost);
6418         device = scmd->device->hostdata;
6419         scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff;
6420
6421         return pqi_device_reset_handler(ctrl_info, device, (u8)scmd->device->lun, scmd, scsi_opcode);
6422 }
6423
6424 static void pqi_tmf_worker(struct work_struct *work)
6425 {
6426         struct pqi_tmf_work *tmf_work;
6427         struct scsi_cmnd *scmd;
6428
6429         tmf_work = container_of(work, struct pqi_tmf_work, work_struct);
6430         scmd = (struct scsi_cmnd *)xchg(&tmf_work->scmd, NULL);
6431
6432         pqi_device_reset_handler(tmf_work->ctrl_info, tmf_work->device, tmf_work->lun, scmd, tmf_work->scsi_opcode);
6433 }
6434
6435 static int pqi_eh_abort_handler(struct scsi_cmnd *scmd)
6436 {
6437         struct Scsi_Host *shost;
6438         struct pqi_ctrl_info *ctrl_info;
6439         struct pqi_scsi_dev *device;
6440         struct pqi_tmf_work *tmf_work;
6441         DECLARE_COMPLETION_ONSTACK(wait);
6442
6443         shost = scmd->device->host;
6444         ctrl_info = shost_to_hba(shost);
6445         device = scmd->device->hostdata;
6446
6447         dev_err(&ctrl_info->pci_dev->dev,
6448                 "attempting TASK ABORT on scsi %d:%d:%d:%d for SCSI cmd at %p\n",
6449                 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6450
6451         if (cmpxchg(&scmd->host_scribble, PQI_NO_COMPLETION, (void *)&wait) == NULL) {
6452                 dev_err(&ctrl_info->pci_dev->dev,
6453                         "scsi %d:%d:%d:%d for SCSI cmd at %p already completed\n",
6454                         shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6455                 scmd->result = DID_RESET << 16;
6456                 goto out;
6457         }
6458
6459         tmf_work = &device->tmf_work[scmd->device->lun];
6460
6461         if (cmpxchg(&tmf_work->scmd, NULL, scmd) == NULL) {
6462                 tmf_work->ctrl_info = ctrl_info;
6463                 tmf_work->device = device;
6464                 tmf_work->lun = (u8)scmd->device->lun;
6465                 tmf_work->scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff;
6466                 schedule_work(&tmf_work->work_struct);
6467         }
6468
6469         wait_for_completion(&wait);
6470
6471         dev_err(&ctrl_info->pci_dev->dev,
6472                 "TASK ABORT on scsi %d:%d:%d:%d for SCSI cmd at %p: SUCCESS\n",
6473                 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6474
6475 out:
6476
6477         return SUCCESS;
6478 }
6479
6480 static int pqi_slave_alloc(struct scsi_device *sdev)
6481 {
6482         struct pqi_scsi_dev *device;
6483         unsigned long flags;
6484         struct pqi_ctrl_info *ctrl_info;
6485         struct scsi_target *starget;
6486         struct sas_rphy *rphy;
6487
6488         ctrl_info = shost_to_hba(sdev->host);
6489
6490         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6491
6492         if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
6493                 starget = scsi_target(sdev);
6494                 rphy = target_to_rphy(starget);
6495                 device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
6496                 if (device) {
6497                         if (device->target_lun_valid) {
6498                                 device->ignore_device = true;
6499                         } else {
6500                                 device->target = sdev_id(sdev);
6501                                 device->lun = sdev->lun;
6502                                 device->target_lun_valid = true;
6503                         }
6504                 }
6505         } else {
6506                 device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
6507                         sdev_id(sdev), sdev->lun);
6508         }
6509
6510         if (device) {
6511                 sdev->hostdata = device;
6512                 device->sdev = sdev;
6513                 if (device->queue_depth) {
6514                         device->advertised_queue_depth = device->queue_depth;
6515                         scsi_change_queue_depth(sdev,
6516                                 device->advertised_queue_depth);
6517                 }
6518                 if (pqi_is_logical_device(device)) {
6519                         pqi_disable_write_same(sdev);
6520                 } else {
6521                         sdev->allow_restart = 1;
6522                         if (device->device_type == SA_DEVICE_TYPE_NVME)
6523                                 pqi_disable_write_same(sdev);
6524                 }
6525         }
6526
6527         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6528
6529         return 0;
6530 }
6531
6532 static void pqi_map_queues(struct Scsi_Host *shost)
6533 {
6534         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6535
6536         if (!ctrl_info->disable_managed_interrupts)
6537                 return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
6538                               ctrl_info->pci_dev, 0);
6539         else
6540                 return blk_mq_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT]);
6541 }
6542
6543 static inline bool pqi_is_tape_changer_device(struct pqi_scsi_dev *device)
6544 {
6545         return device->devtype == TYPE_TAPE || device->devtype == TYPE_MEDIUM_CHANGER;
6546 }
6547
6548 static int pqi_slave_configure(struct scsi_device *sdev)
6549 {
6550         int rc = 0;
6551         struct pqi_scsi_dev *device;
6552
6553         device = sdev->hostdata;
6554         device->devtype = sdev->type;
6555
6556         if (pqi_is_tape_changer_device(device) && device->ignore_device) {
6557                 rc = -ENXIO;
6558                 device->ignore_device = false;
6559         }
6560
6561         return rc;
6562 }
6563
6564 static void pqi_slave_destroy(struct scsi_device *sdev)
6565 {
6566         struct pqi_ctrl_info *ctrl_info;
6567         struct pqi_scsi_dev *device;
6568         int mutex_acquired;
6569         unsigned long flags;
6570
6571         ctrl_info = shost_to_hba(sdev->host);
6572
6573         mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);
6574         if (!mutex_acquired)
6575                 return;
6576
6577         device = sdev->hostdata;
6578         if (!device) {
6579                 mutex_unlock(&ctrl_info->scan_mutex);
6580                 return;
6581         }
6582
6583         device->lun_count--;
6584         if (device->lun_count > 0) {
6585                 mutex_unlock(&ctrl_info->scan_mutex);
6586                 return;
6587         }
6588
6589         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6590         list_del(&device->scsi_device_list_entry);
6591         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6592
6593         mutex_unlock(&ctrl_info->scan_mutex);
6594
6595         pqi_dev_info(ctrl_info, "removed", device);
6596         pqi_free_device(device);
6597 }
6598
6599 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6600 {
6601         struct pci_dev *pci_dev;
6602         u32 subsystem_vendor;
6603         u32 subsystem_device;
6604         cciss_pci_info_struct pci_info;
6605
6606         if (!arg)
6607                 return -EINVAL;
6608
6609         pci_dev = ctrl_info->pci_dev;
6610
6611         pci_info.domain = pci_domain_nr(pci_dev->bus);
6612         pci_info.bus = pci_dev->bus->number;
6613         pci_info.dev_fn = pci_dev->devfn;
6614         subsystem_vendor = pci_dev->subsystem_vendor;
6615         subsystem_device = pci_dev->subsystem_device;
6616         pci_info.board_id = ((subsystem_device << 16) & 0xffff0000) | subsystem_vendor;
6617
6618         if (copy_to_user(arg, &pci_info, sizeof(pci_info)))
6619                 return -EFAULT;
6620
6621         return 0;
6622 }
6623
6624 static int pqi_getdrivver_ioctl(void __user *arg)
6625 {
6626         u32 version;
6627
6628         if (!arg)
6629                 return -EINVAL;
6630
6631         version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
6632                 (DRIVER_RELEASE << 16) | DRIVER_REVISION;
6633
6634         if (copy_to_user(arg, &version, sizeof(version)))
6635                 return -EFAULT;
6636
6637         return 0;
6638 }
6639
6640 struct ciss_error_info {
6641         u8      scsi_status;
6642         int     command_status;
6643         size_t  sense_data_length;
6644 };
6645
6646 static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
6647         struct ciss_error_info *ciss_error_info)
6648 {
6649         int ciss_cmd_status;
6650         size_t sense_data_length;
6651
6652         switch (pqi_error_info->data_out_result) {
6653         case PQI_DATA_IN_OUT_GOOD:
6654                 ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
6655                 break;
6656         case PQI_DATA_IN_OUT_UNDERFLOW:
6657                 ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
6658                 break;
6659         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
6660                 ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
6661                 break;
6662         case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
6663         case PQI_DATA_IN_OUT_BUFFER_ERROR:
6664         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
6665         case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
6666         case PQI_DATA_IN_OUT_ERROR:
6667                 ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
6668                 break;
6669         case PQI_DATA_IN_OUT_HARDWARE_ERROR:
6670         case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
6671         case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
6672         case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
6673         case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
6674         case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
6675         case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
6676         case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
6677         case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
6678         case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
6679                 ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
6680                 break;
6681         case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
6682                 ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
6683                 break;
6684         case PQI_DATA_IN_OUT_ABORTED:
6685                 ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
6686                 break;
6687         case PQI_DATA_IN_OUT_TIMEOUT:
6688                 ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
6689                 break;
6690         default:
6691                 ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
6692                 break;
6693         }
6694
6695         sense_data_length =
6696                 get_unaligned_le16(&pqi_error_info->sense_data_length);
6697         if (sense_data_length == 0)
6698                 sense_data_length =
6699                 get_unaligned_le16(&pqi_error_info->response_data_length);
6700         if (sense_data_length)
6701                 if (sense_data_length > sizeof(pqi_error_info->data))
6702                         sense_data_length = sizeof(pqi_error_info->data);
6703
6704         ciss_error_info->scsi_status = pqi_error_info->status;
6705         ciss_error_info->command_status = ciss_cmd_status;
6706         ciss_error_info->sense_data_length = sense_data_length;
6707 }
6708
6709 static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6710 {
6711         int rc;
6712         char *kernel_buffer = NULL;
6713         u16 iu_length;
6714         size_t sense_data_length;
6715         IOCTL_Command_struct iocommand;
6716         struct pqi_raid_path_request request;
6717         struct pqi_raid_error_info pqi_error_info;
6718         struct ciss_error_info ciss_error_info;
6719
6720         if (pqi_ctrl_offline(ctrl_info))
6721                 return -ENXIO;
6722         if (pqi_ofa_in_progress(ctrl_info) && pqi_ctrl_blocked(ctrl_info))
6723                 return -EBUSY;
6724         if (!arg)
6725                 return -EINVAL;
6726         if (!capable(CAP_SYS_RAWIO))
6727                 return -EPERM;
6728         if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
6729                 return -EFAULT;
6730         if (iocommand.buf_size < 1 &&
6731                 iocommand.Request.Type.Direction != XFER_NONE)
6732                 return -EINVAL;
6733         if (iocommand.Request.CDBLen > sizeof(request.cdb))
6734                 return -EINVAL;
6735         if (iocommand.Request.Type.Type != TYPE_CMD)
6736                 return -EINVAL;
6737
6738         switch (iocommand.Request.Type.Direction) {
6739         case XFER_NONE:
6740         case XFER_WRITE:
6741         case XFER_READ:
6742         case XFER_READ | XFER_WRITE:
6743                 break;
6744         default:
6745                 return -EINVAL;
6746         }
6747
6748         if (iocommand.buf_size > 0) {
6749                 kernel_buffer = kmalloc(iocommand.buf_size, GFP_KERNEL);
6750                 if (!kernel_buffer)
6751                         return -ENOMEM;
6752                 if (iocommand.Request.Type.Direction & XFER_WRITE) {
6753                         if (copy_from_user(kernel_buffer, iocommand.buf,
6754                                 iocommand.buf_size)) {
6755                                 rc = -EFAULT;
6756                                 goto out;
6757                         }
6758                 } else {
6759                         memset(kernel_buffer, 0, iocommand.buf_size);
6760                 }
6761         }
6762
6763         memset(&request, 0, sizeof(request));
6764
6765         request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
6766         iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
6767                 PQI_REQUEST_HEADER_LENGTH;
6768         memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
6769                 sizeof(request.lun_number));
6770         memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
6771         request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
6772
6773         switch (iocommand.Request.Type.Direction) {
6774         case XFER_NONE:
6775                 request.data_direction = SOP_NO_DIRECTION_FLAG;
6776                 break;
6777         case XFER_WRITE:
6778                 request.data_direction = SOP_WRITE_FLAG;
6779                 break;
6780         case XFER_READ:
6781                 request.data_direction = SOP_READ_FLAG;
6782                 break;
6783         case XFER_READ | XFER_WRITE:
6784                 request.data_direction = SOP_BIDIRECTIONAL;
6785                 break;
6786         }
6787
6788         request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
6789
6790         if (iocommand.buf_size > 0) {
6791                 put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
6792
6793                 rc = pqi_map_single(ctrl_info->pci_dev,
6794                         &request.sg_descriptors[0], kernel_buffer,
6795                         iocommand.buf_size, DMA_BIDIRECTIONAL);
6796                 if (rc)
6797                         goto out;
6798
6799                 iu_length += sizeof(request.sg_descriptors[0]);
6800         }
6801
6802         put_unaligned_le16(iu_length, &request.header.iu_length);
6803
6804         if (ctrl_info->raid_iu_timeout_supported)
6805                 put_unaligned_le32(iocommand.Request.Timeout, &request.timeout);
6806
6807         rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
6808                 PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info);
6809
6810         if (iocommand.buf_size > 0)
6811                 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
6812                         DMA_BIDIRECTIONAL);
6813
6814         memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
6815
6816         if (rc == 0) {
6817                 pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
6818                 iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
6819                 iocommand.error_info.CommandStatus =
6820                         ciss_error_info.command_status;
6821                 sense_data_length = ciss_error_info.sense_data_length;
6822                 if (sense_data_length) {
6823                         if (sense_data_length >
6824                                 sizeof(iocommand.error_info.SenseInfo))
6825                                 sense_data_length =
6826                                         sizeof(iocommand.error_info.SenseInfo);
6827                         memcpy(iocommand.error_info.SenseInfo,
6828                                 pqi_error_info.data, sense_data_length);
6829                         iocommand.error_info.SenseLen = sense_data_length;
6830                 }
6831         }
6832
6833         if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
6834                 rc = -EFAULT;
6835                 goto out;
6836         }
6837
6838         if (rc == 0 && iocommand.buf_size > 0 &&
6839                 (iocommand.Request.Type.Direction & XFER_READ)) {
6840                 if (copy_to_user(iocommand.buf, kernel_buffer,
6841                         iocommand.buf_size)) {
6842                         rc = -EFAULT;
6843                 }
6844         }
6845
6846 out:
6847         kfree(kernel_buffer);
6848
6849         return rc;
6850 }
6851
6852 static int pqi_ioctl(struct scsi_device *sdev, unsigned int cmd,
6853                      void __user *arg)
6854 {
6855         int rc;
6856         struct pqi_ctrl_info *ctrl_info;
6857
6858         ctrl_info = shost_to_hba(sdev->host);
6859
6860         switch (cmd) {
6861         case CCISS_DEREGDISK:
6862         case CCISS_REGNEWDISK:
6863         case CCISS_REGNEWD:
6864                 rc = pqi_scan_scsi_devices(ctrl_info);
6865                 break;
6866         case CCISS_GETPCIINFO:
6867                 rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
6868                 break;
6869         case CCISS_GETDRIVVER:
6870                 rc = pqi_getdrivver_ioctl(arg);
6871                 break;
6872         case CCISS_PASSTHRU:
6873                 rc = pqi_passthru_ioctl(ctrl_info, arg);
6874                 break;
6875         default:
6876                 rc = -EINVAL;
6877                 break;
6878         }
6879
6880         return rc;
6881 }
6882
6883 static ssize_t pqi_firmware_version_show(struct device *dev,
6884         struct device_attribute *attr, char *buffer)
6885 {
6886         struct Scsi_Host *shost;
6887         struct pqi_ctrl_info *ctrl_info;
6888
6889         shost = class_to_shost(dev);
6890         ctrl_info = shost_to_hba(shost);
6891
6892         return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);
6893 }
6894
6895 static ssize_t pqi_driver_version_show(struct device *dev,
6896         struct device_attribute *attr, char *buffer)
6897 {
6898         return scnprintf(buffer, PAGE_SIZE, "%s\n", DRIVER_VERSION BUILD_TIMESTAMP);
6899 }
6900
6901 static ssize_t pqi_serial_number_show(struct device *dev,
6902         struct device_attribute *attr, char *buffer)
6903 {
6904         struct Scsi_Host *shost;
6905         struct pqi_ctrl_info *ctrl_info;
6906
6907         shost = class_to_shost(dev);
6908         ctrl_info = shost_to_hba(shost);
6909
6910         return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);
6911 }
6912
6913 static ssize_t pqi_model_show(struct device *dev,
6914         struct device_attribute *attr, char *buffer)
6915 {
6916         struct Scsi_Host *shost;
6917         struct pqi_ctrl_info *ctrl_info;
6918
6919         shost = class_to_shost(dev);
6920         ctrl_info = shost_to_hba(shost);
6921
6922         return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);
6923 }
6924
6925 static ssize_t pqi_vendor_show(struct device *dev,
6926         struct device_attribute *attr, char *buffer)
6927 {
6928         struct Scsi_Host *shost;
6929         struct pqi_ctrl_info *ctrl_info;
6930
6931         shost = class_to_shost(dev);
6932         ctrl_info = shost_to_hba(shost);
6933
6934         return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);
6935 }
6936
6937 static ssize_t pqi_host_rescan_store(struct device *dev,
6938         struct device_attribute *attr, const char *buffer, size_t count)
6939 {
6940         struct Scsi_Host *shost = class_to_shost(dev);
6941
6942         pqi_scan_start(shost);
6943
6944         return count;
6945 }
6946
6947 static ssize_t pqi_lockup_action_show(struct device *dev,
6948         struct device_attribute *attr, char *buffer)
6949 {
6950         int count = 0;
6951         unsigned int i;
6952
6953         for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6954                 if (pqi_lockup_actions[i].action == pqi_lockup_action)
6955                         count += scnprintf(buffer + count, PAGE_SIZE - count,
6956                                 "[%s] ", pqi_lockup_actions[i].name);
6957                 else
6958                         count += scnprintf(buffer + count, PAGE_SIZE - count,
6959                                 "%s ", pqi_lockup_actions[i].name);
6960         }
6961
6962         count += scnprintf(buffer + count, PAGE_SIZE - count, "\n");
6963
6964         return count;
6965 }
6966
6967 static ssize_t pqi_lockup_action_store(struct device *dev,
6968         struct device_attribute *attr, const char *buffer, size_t count)
6969 {
6970         unsigned int i;
6971         char *action_name;
6972         char action_name_buffer[32];
6973
6974         strscpy(action_name_buffer, buffer, sizeof(action_name_buffer));
6975         action_name = strstrip(action_name_buffer);
6976
6977         for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6978                 if (strcmp(action_name, pqi_lockup_actions[i].name) == 0) {
6979                         pqi_lockup_action = pqi_lockup_actions[i].action;
6980                         return count;
6981                 }
6982         }
6983
6984         return -EINVAL;
6985 }
6986
6987 static ssize_t pqi_host_enable_stream_detection_show(struct device *dev,
6988         struct device_attribute *attr, char *buffer)
6989 {
6990         struct Scsi_Host *shost = class_to_shost(dev);
6991         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6992
6993         return scnprintf(buffer, 10, "%x\n",
6994                         ctrl_info->enable_stream_detection);
6995 }
6996
6997 static ssize_t pqi_host_enable_stream_detection_store(struct device *dev,
6998         struct device_attribute *attr, const char *buffer, size_t count)
6999 {
7000         struct Scsi_Host *shost = class_to_shost(dev);
7001         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7002         u8 set_stream_detection = 0;
7003
7004         if (kstrtou8(buffer, 0, &set_stream_detection))
7005                 return -EINVAL;
7006
7007         if (set_stream_detection > 0)
7008                 set_stream_detection = 1;
7009
7010         ctrl_info->enable_stream_detection = set_stream_detection;
7011
7012         return count;
7013 }
7014
7015 static ssize_t pqi_host_enable_r5_writes_show(struct device *dev,
7016         struct device_attribute *attr, char *buffer)
7017 {
7018         struct Scsi_Host *shost = class_to_shost(dev);
7019         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7020
7021         return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r5_writes);
7022 }
7023
7024 static ssize_t pqi_host_enable_r5_writes_store(struct device *dev,
7025         struct device_attribute *attr, const char *buffer, size_t count)
7026 {
7027         struct Scsi_Host *shost = class_to_shost(dev);
7028         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7029         u8 set_r5_writes = 0;
7030
7031         if (kstrtou8(buffer, 0, &set_r5_writes))
7032                 return -EINVAL;
7033
7034         if (set_r5_writes > 0)
7035                 set_r5_writes = 1;
7036
7037         ctrl_info->enable_r5_writes = set_r5_writes;
7038
7039         return count;
7040 }
7041
7042 static ssize_t pqi_host_enable_r6_writes_show(struct device *dev,
7043         struct device_attribute *attr, char *buffer)
7044 {
7045         struct Scsi_Host *shost = class_to_shost(dev);
7046         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7047
7048         return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r6_writes);
7049 }
7050
7051 static ssize_t pqi_host_enable_r6_writes_store(struct device *dev,
7052         struct device_attribute *attr, const char *buffer, size_t count)
7053 {
7054         struct Scsi_Host *shost = class_to_shost(dev);
7055         struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7056         u8 set_r6_writes = 0;
7057
7058         if (kstrtou8(buffer, 0, &set_r6_writes))
7059                 return -EINVAL;
7060
7061         if (set_r6_writes > 0)
7062                 set_r6_writes = 1;
7063
7064         ctrl_info->enable_r6_writes = set_r6_writes;
7065
7066         return count;
7067 }
7068
7069 static DEVICE_ATTR(driver_version, 0444, pqi_driver_version_show, NULL);
7070 static DEVICE_ATTR(firmware_version, 0444, pqi_firmware_version_show, NULL);
7071 static DEVICE_ATTR(model, 0444, pqi_model_show, NULL);
7072 static DEVICE_ATTR(serial_number, 0444, pqi_serial_number_show, NULL);
7073 static DEVICE_ATTR(vendor, 0444, pqi_vendor_show, NULL);
7074 static DEVICE_ATTR(rescan, 0200, NULL, pqi_host_rescan_store);
7075 static DEVICE_ATTR(lockup_action, 0644, pqi_lockup_action_show,
7076         pqi_lockup_action_store);
7077 static DEVICE_ATTR(enable_stream_detection, 0644,
7078         pqi_host_enable_stream_detection_show,
7079         pqi_host_enable_stream_detection_store);
7080 static DEVICE_ATTR(enable_r5_writes, 0644,
7081         pqi_host_enable_r5_writes_show, pqi_host_enable_r5_writes_store);
7082 static DEVICE_ATTR(enable_r6_writes, 0644,
7083         pqi_host_enable_r6_writes_show, pqi_host_enable_r6_writes_store);
7084
7085 static struct attribute *pqi_shost_attrs[] = {
7086         &dev_attr_driver_version.attr,
7087         &dev_attr_firmware_version.attr,
7088         &dev_attr_model.attr,
7089         &dev_attr_serial_number.attr,
7090         &dev_attr_vendor.attr,
7091         &dev_attr_rescan.attr,
7092         &dev_attr_lockup_action.attr,
7093         &dev_attr_enable_stream_detection.attr,
7094         &dev_attr_enable_r5_writes.attr,
7095         &dev_attr_enable_r6_writes.attr,
7096         NULL
7097 };
7098
7099 ATTRIBUTE_GROUPS(pqi_shost);
7100
7101 static ssize_t pqi_unique_id_show(struct device *dev,
7102         struct device_attribute *attr, char *buffer)
7103 {
7104         struct pqi_ctrl_info *ctrl_info;
7105         struct scsi_device *sdev;
7106         struct pqi_scsi_dev *device;
7107         unsigned long flags;
7108         u8 unique_id[16];
7109
7110         sdev = to_scsi_device(dev);
7111         ctrl_info = shost_to_hba(sdev->host);
7112
7113         if (pqi_ctrl_offline(ctrl_info))
7114                 return -ENODEV;
7115
7116         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7117
7118         device = sdev->hostdata;
7119         if (!device) {
7120                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7121                 return -ENODEV;
7122         }
7123
7124         if (device->is_physical_device)
7125                 memcpy(unique_id, device->wwid, sizeof(device->wwid));
7126         else
7127                 memcpy(unique_id, device->volume_id, sizeof(device->volume_id));
7128
7129         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7130
7131         return scnprintf(buffer, PAGE_SIZE,
7132                 "%02X%02X%02X%02X%02X%02X%02X%02X"
7133                 "%02X%02X%02X%02X%02X%02X%02X%02X\n",
7134                 unique_id[0], unique_id[1], unique_id[2], unique_id[3],
7135                 unique_id[4], unique_id[5], unique_id[6], unique_id[7],
7136                 unique_id[8], unique_id[9], unique_id[10], unique_id[11],
7137                 unique_id[12], unique_id[13], unique_id[14], unique_id[15]);
7138 }
7139
7140 static ssize_t pqi_lunid_show(struct device *dev,
7141         struct device_attribute *attr, char *buffer)
7142 {
7143         struct pqi_ctrl_info *ctrl_info;
7144         struct scsi_device *sdev;
7145         struct pqi_scsi_dev *device;
7146         unsigned long flags;
7147         u8 lunid[8];
7148
7149         sdev = to_scsi_device(dev);
7150         ctrl_info = shost_to_hba(sdev->host);
7151
7152         if (pqi_ctrl_offline(ctrl_info))
7153                 return -ENODEV;
7154
7155         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7156
7157         device = sdev->hostdata;
7158         if (!device) {
7159                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7160                 return -ENODEV;
7161         }
7162
7163         memcpy(lunid, device->scsi3addr, sizeof(lunid));
7164
7165         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7166
7167         return scnprintf(buffer, PAGE_SIZE, "0x%8phN\n", lunid);
7168 }
7169
7170 #define MAX_PATHS       8
7171
7172 static ssize_t pqi_path_info_show(struct device *dev,
7173         struct device_attribute *attr, char *buf)
7174 {
7175         struct pqi_ctrl_info *ctrl_info;
7176         struct scsi_device *sdev;
7177         struct pqi_scsi_dev *device;
7178         unsigned long flags;
7179         int i;
7180         int output_len = 0;
7181         u8 box;
7182         u8 bay;
7183         u8 path_map_index;
7184         char *active;
7185         u8 phys_connector[2];
7186
7187         sdev = to_scsi_device(dev);
7188         ctrl_info = shost_to_hba(sdev->host);
7189
7190         if (pqi_ctrl_offline(ctrl_info))
7191                 return -ENODEV;
7192
7193         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7194
7195         device = sdev->hostdata;
7196         if (!device) {
7197                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7198                 return -ENODEV;
7199         }
7200
7201         bay = device->bay;
7202         for (i = 0; i < MAX_PATHS; i++) {
7203                 path_map_index = 1 << i;
7204                 if (i == device->active_path_index)
7205                         active = "Active";
7206                 else if (device->path_map & path_map_index)
7207                         active = "Inactive";
7208                 else
7209                         continue;
7210
7211                 output_len += scnprintf(buf + output_len,
7212                                         PAGE_SIZE - output_len,
7213                                         "[%d:%d:%d:%d] %20.20s ",
7214                                         ctrl_info->scsi_host->host_no,
7215                                         device->bus, device->target,
7216                                         device->lun,
7217                                         scsi_device_type(device->devtype));
7218
7219                 if (device->devtype == TYPE_RAID ||
7220                         pqi_is_logical_device(device))
7221                         goto end_buffer;
7222
7223                 memcpy(&phys_connector, &device->phys_connector[i],
7224                         sizeof(phys_connector));
7225                 if (phys_connector[0] < '0')
7226                         phys_connector[0] = '0';
7227                 if (phys_connector[1] < '0')
7228                         phys_connector[1] = '0';
7229
7230                 output_len += scnprintf(buf + output_len,
7231                                         PAGE_SIZE - output_len,
7232                                         "PORT: %.2s ", phys_connector);
7233
7234                 box = device->box[i];
7235                 if (box != 0 && box != 0xFF)
7236                         output_len += scnprintf(buf + output_len,
7237                                                 PAGE_SIZE - output_len,
7238                                                 "BOX: %hhu ", box);
7239
7240                 if ((device->devtype == TYPE_DISK ||
7241                         device->devtype == TYPE_ZBC) &&
7242                         pqi_expose_device(device))
7243                         output_len += scnprintf(buf + output_len,
7244                                                 PAGE_SIZE - output_len,
7245                                                 "BAY: %hhu ", bay);
7246
7247 end_buffer:
7248                 output_len += scnprintf(buf + output_len,
7249                                         PAGE_SIZE - output_len,
7250                                         "%s\n", active);
7251         }
7252
7253         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7254
7255         return output_len;
7256 }
7257
7258 static ssize_t pqi_sas_address_show(struct device *dev,
7259         struct device_attribute *attr, char *buffer)
7260 {
7261         struct pqi_ctrl_info *ctrl_info;
7262         struct scsi_device *sdev;
7263         struct pqi_scsi_dev *device;
7264         unsigned long flags;
7265         u64 sas_address;
7266
7267         sdev = to_scsi_device(dev);
7268         ctrl_info = shost_to_hba(sdev->host);
7269
7270         if (pqi_ctrl_offline(ctrl_info))
7271                 return -ENODEV;
7272
7273         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7274
7275         device = sdev->hostdata;
7276         if (!device) {
7277                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7278                 return -ENODEV;
7279         }
7280
7281         sas_address = device->sas_address;
7282
7283         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7284
7285         return scnprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
7286 }
7287
7288 static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
7289         struct device_attribute *attr, char *buffer)
7290 {
7291         struct pqi_ctrl_info *ctrl_info;
7292         struct scsi_device *sdev;
7293         struct pqi_scsi_dev *device;
7294         unsigned long flags;
7295
7296         sdev = to_scsi_device(dev);
7297         ctrl_info = shost_to_hba(sdev->host);
7298
7299         if (pqi_ctrl_offline(ctrl_info))
7300                 return -ENODEV;
7301
7302         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7303
7304         device = sdev->hostdata;
7305         if (!device) {
7306                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7307                 return -ENODEV;
7308         }
7309
7310         buffer[0] = device->raid_bypass_enabled ? '1' : '0';
7311         buffer[1] = '\n';
7312         buffer[2] = '\0';
7313
7314         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7315
7316         return 2;
7317 }
7318
7319 static ssize_t pqi_raid_level_show(struct device *dev,
7320         struct device_attribute *attr, char *buffer)
7321 {
7322         struct pqi_ctrl_info *ctrl_info;
7323         struct scsi_device *sdev;
7324         struct pqi_scsi_dev *device;
7325         unsigned long flags;
7326         char *raid_level;
7327
7328         sdev = to_scsi_device(dev);
7329         ctrl_info = shost_to_hba(sdev->host);
7330
7331         if (pqi_ctrl_offline(ctrl_info))
7332                 return -ENODEV;
7333
7334         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7335
7336         device = sdev->hostdata;
7337         if (!device) {
7338                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7339                 return -ENODEV;
7340         }
7341
7342         if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK)
7343                 raid_level = pqi_raid_level_to_string(device->raid_level);
7344         else
7345                 raid_level = "N/A";
7346
7347         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7348
7349         return scnprintf(buffer, PAGE_SIZE, "%s\n", raid_level);
7350 }
7351
7352 static ssize_t pqi_raid_bypass_cnt_show(struct device *dev,
7353         struct device_attribute *attr, char *buffer)
7354 {
7355         struct pqi_ctrl_info *ctrl_info;
7356         struct scsi_device *sdev;
7357         struct pqi_scsi_dev *device;
7358         unsigned long flags;
7359         unsigned int raid_bypass_cnt;
7360
7361         sdev = to_scsi_device(dev);
7362         ctrl_info = shost_to_hba(sdev->host);
7363
7364         if (pqi_ctrl_offline(ctrl_info))
7365                 return -ENODEV;
7366
7367         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7368
7369         device = sdev->hostdata;
7370         if (!device) {
7371                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7372                 return -ENODEV;
7373         }
7374
7375         raid_bypass_cnt = device->raid_bypass_cnt;
7376
7377         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7378
7379         return scnprintf(buffer, PAGE_SIZE, "0x%x\n", raid_bypass_cnt);
7380 }
7381
7382 static ssize_t pqi_sas_ncq_prio_enable_show(struct device *dev,
7383                 struct device_attribute *attr, char *buf)
7384 {
7385         struct pqi_ctrl_info *ctrl_info;
7386         struct scsi_device *sdev;
7387         struct pqi_scsi_dev *device;
7388         unsigned long flags;
7389         int output_len = 0;
7390
7391         sdev = to_scsi_device(dev);
7392         ctrl_info = shost_to_hba(sdev->host);
7393
7394         if (pqi_ctrl_offline(ctrl_info))
7395                 return -ENODEV;
7396
7397         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7398
7399         device = sdev->hostdata;
7400         if (!device) {
7401                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7402                 return -ENODEV;
7403         }
7404
7405         output_len = snprintf(buf, PAGE_SIZE, "%d\n",
7406                                 device->ncq_prio_enable);
7407         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7408
7409         return output_len;
7410 }
7411
7412 static ssize_t pqi_sas_ncq_prio_enable_store(struct device *dev,
7413                         struct device_attribute *attr,
7414                         const char *buf, size_t count)
7415 {
7416         struct pqi_ctrl_info *ctrl_info;
7417         struct scsi_device *sdev;
7418         struct pqi_scsi_dev *device;
7419         unsigned long flags;
7420         u8 ncq_prio_enable = 0;
7421
7422         if (kstrtou8(buf, 0, &ncq_prio_enable))
7423                 return -EINVAL;
7424
7425         sdev = to_scsi_device(dev);
7426         ctrl_info = shost_to_hba(sdev->host);
7427
7428         spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7429
7430         device = sdev->hostdata;
7431
7432         if (!device) {
7433                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7434                 return -ENODEV;
7435         }
7436
7437         if (!device->ncq_prio_support) {
7438                 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7439                 return -EINVAL;
7440         }
7441
7442         device->ncq_prio_enable = ncq_prio_enable;
7443
7444         spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7445
7446         return  strlen(buf);
7447 }
7448
7449 static ssize_t pqi_numa_node_show(struct device *dev,
7450         struct device_attribute *attr, char *buffer)
7451 {
7452         struct scsi_device *sdev;
7453         struct pqi_ctrl_info *ctrl_info;
7454
7455         sdev = to_scsi_device(dev);
7456         ctrl_info = shost_to_hba(sdev->host);
7457
7458         return scnprintf(buffer, PAGE_SIZE, "%d\n", ctrl_info->numa_node);
7459 }
7460
7461 static DEVICE_ATTR(lunid, 0444, pqi_lunid_show, NULL);
7462 static DEVICE_ATTR(unique_id, 0444, pqi_unique_id_show, NULL);
7463 static DEVICE_ATTR(path_info, 0444, pqi_path_info_show, NULL);
7464 static DEVICE_ATTR(sas_address, 0444, pqi_sas_address_show, NULL);
7465 static DEVICE_ATTR(ssd_smart_path_enabled, 0444, pqi_ssd_smart_path_enabled_show, NULL);
7466 static DEVICE_ATTR(raid_level, 0444, pqi_raid_level_show, NULL);
7467 static DEVICE_ATTR(raid_bypass_cnt, 0444, pqi_raid_bypass_cnt_show, NULL);
7468 static DEVICE_ATTR(sas_ncq_prio_enable, 0644,
7469                 pqi_sas_ncq_prio_enable_show, pqi_sas_ncq_prio_enable_store);
7470 static DEVICE_ATTR(numa_node, 0444, pqi_numa_node_show, NULL);
7471
7472 static struct attribute *pqi_sdev_attrs[] = {
7473         &dev_attr_lunid.attr,
7474         &dev_attr_unique_id.attr,
7475         &dev_attr_path_info.attr,
7476         &dev_attr_sas_address.attr,
7477         &dev_attr_ssd_smart_path_enabled.attr,
7478         &dev_attr_raid_level.attr,
7479         &dev_attr_raid_bypass_cnt.attr,
7480         &dev_attr_sas_ncq_prio_enable.attr,
7481         &dev_attr_numa_node.attr,
7482         NULL
7483 };
7484
7485 ATTRIBUTE_GROUPS(pqi_sdev);
7486
7487 static const struct scsi_host_template pqi_driver_template = {
7488         .module = THIS_MODULE,
7489         .name = DRIVER_NAME_SHORT,
7490         .proc_name = DRIVER_NAME_SHORT,
7491         .queuecommand = pqi_scsi_queue_command,
7492         .scan_start = pqi_scan_start,
7493         .scan_finished = pqi_scan_finished,
7494         .this_id = -1,
7495         .eh_device_reset_handler = pqi_eh_device_reset_handler,
7496         .eh_abort_handler = pqi_eh_abort_handler,
7497         .ioctl = pqi_ioctl,
7498         .slave_alloc = pqi_slave_alloc,
7499         .slave_configure = pqi_slave_configure,
7500         .slave_destroy = pqi_slave_destroy,
7501         .map_queues = pqi_map_queues,
7502         .sdev_groups = pqi_sdev_groups,
7503         .shost_groups = pqi_shost_groups,
7504         .cmd_size = sizeof(struct pqi_cmd_priv),
7505 };
7506
7507 static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
7508 {
7509         int rc;
7510         struct Scsi_Host *shost;
7511
7512         shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
7513         if (!shost) {
7514                 dev_err(&ctrl_info->pci_dev->dev, "scsi_host_alloc failed\n");
7515                 return -ENOMEM;
7516         }
7517
7518         shost->io_port = 0;
7519         shost->n_io_port = 0;
7520         shost->this_id = -1;
7521         shost->max_channel = PQI_MAX_BUS;
7522         shost->max_cmd_len = MAX_COMMAND_SIZE;
7523         shost->max_lun = PQI_MAX_LUNS_PER_DEVICE;
7524         shost->max_id = ~0;
7525         shost->max_sectors = ctrl_info->max_sectors;
7526         shost->can_queue = ctrl_info->scsi_ml_can_queue;
7527         shost->cmd_per_lun = shost->can_queue;
7528         shost->sg_tablesize = ctrl_info->sg_tablesize;
7529         shost->transportt = pqi_sas_transport_template;
7530         shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
7531         shost->unique_id = shost->irq;
7532         shost->nr_hw_queues = ctrl_info->num_queue_groups;
7533         shost->host_tagset = 1;
7534         shost->hostdata[0] = (unsigned long)ctrl_info;
7535
7536         rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
7537         if (rc) {
7538                 dev_err(&ctrl_info->pci_dev->dev, "scsi_add_host failed\n");
7539                 goto free_host;
7540         }
7541
7542         rc = pqi_add_sas_host(shost, ctrl_info);
7543         if (rc) {
7544                 dev_err(&ctrl_info->pci_dev->dev, "add SAS host failed\n");
7545                 goto remove_host;
7546         }
7547
7548         ctrl_info->scsi_host = shost;
7549
7550         return 0;
7551
7552 remove_host:
7553         scsi_remove_host(shost);
7554 free_host:
7555         scsi_host_put(shost);
7556
7557         return rc;
7558 }
7559
7560 static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
7561 {
7562         struct Scsi_Host *shost;
7563
7564         pqi_delete_sas_host(ctrl_info);
7565
7566         shost = ctrl_info->scsi_host;
7567         if (!shost)
7568                 return;
7569
7570         scsi_remove_host(shost);
7571         scsi_host_put(shost);
7572 }
7573
7574 static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)
7575 {
7576         int rc = 0;
7577         struct pqi_device_registers __iomem *pqi_registers;
7578         unsigned long timeout;
7579         unsigned int timeout_msecs;
7580         union pqi_reset_register reset_reg;
7581
7582         pqi_registers = ctrl_info->pqi_registers;
7583         timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100;
7584         timeout = msecs_to_jiffies(timeout_msecs) + jiffies;
7585
7586         while (1) {
7587                 msleep(PQI_RESET_POLL_INTERVAL_MSECS);
7588                 reset_reg.all_bits = readl(&pqi_registers->device_reset);
7589                 if (reset_reg.bits.reset_action == PQI_RESET_ACTION_COMPLETED)
7590                         break;
7591                 if (!sis_is_firmware_running(ctrl_info)) {
7592                         rc = -ENXIO;
7593                         break;
7594                 }
7595                 if (time_after(jiffies, timeout)) {
7596                         rc = -ETIMEDOUT;
7597                         break;
7598                 }
7599         }
7600
7601         return rc;
7602 }
7603
7604 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
7605 {
7606         int rc;
7607         union pqi_reset_register reset_reg;
7608
7609         if (ctrl_info->pqi_reset_quiesce_supported) {
7610                 rc = sis_pqi_reset_quiesce(ctrl_info);
7611                 if (rc) {
7612                         dev_err(&ctrl_info->pci_dev->dev,
7613                                 "PQI reset failed during quiesce with error %d\n", rc);
7614                         return rc;
7615                 }
7616         }
7617
7618         reset_reg.all_bits = 0;
7619         reset_reg.bits.reset_type = PQI_RESET_TYPE_HARD_RESET;
7620         reset_reg.bits.reset_action = PQI_RESET_ACTION_RESET;
7621
7622         writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);
7623
7624         rc = pqi_wait_for_pqi_reset_completion(ctrl_info);
7625         if (rc)
7626                 dev_err(&ctrl_info->pci_dev->dev,
7627                         "PQI reset failed with error %d\n", rc);
7628
7629         return rc;
7630 }
7631
7632 static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)
7633 {
7634         int rc;
7635         struct bmic_sense_subsystem_info *sense_info;
7636
7637         sense_info = kzalloc(sizeof(*sense_info), GFP_KERNEL);
7638         if (!sense_info)
7639                 return -ENOMEM;
7640
7641         rc = pqi_sense_subsystem_info(ctrl_info, sense_info);
7642         if (rc)
7643                 goto out;
7644
7645         memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,
7646                 sizeof(sense_info->ctrl_serial_number));
7647         ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';
7648
7649 out:
7650         kfree(sense_info);
7651
7652         return rc;
7653 }
7654
7655 static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)
7656 {
7657         int rc;
7658         struct bmic_identify_controller *identify;
7659
7660         identify = kmalloc(sizeof(*identify), GFP_KERNEL);
7661         if (!identify)
7662                 return -ENOMEM;
7663
7664         rc = pqi_identify_controller(ctrl_info, identify);
7665         if (rc)
7666                 goto out;
7667
7668         if (get_unaligned_le32(&identify->extra_controller_flags) &
7669                 BMIC_IDENTIFY_EXTRA_FLAGS_LONG_FW_VERSION_SUPPORTED) {
7670                 memcpy(ctrl_info->firmware_version,
7671                         identify->firmware_version_long,
7672                         sizeof(identify->firmware_version_long));
7673         } else {
7674                 memcpy(ctrl_info->firmware_version,
7675                         identify->firmware_version_short,
7676                         sizeof(identify->firmware_version_short));
7677                 ctrl_info->firmware_version
7678                         [sizeof(identify->firmware_version_short)] = '\0';
7679                 snprintf(ctrl_info->firmware_version +
7680                         strlen(ctrl_info->firmware_version),
7681                         sizeof(ctrl_info->firmware_version) -
7682                         sizeof(identify->firmware_version_short),
7683                         "-%u",
7684                         get_unaligned_le16(&identify->firmware_build_number));
7685         }
7686
7687         memcpy(ctrl_info->model, identify->product_id,
7688                 sizeof(identify->product_id));
7689         ctrl_info->model[sizeof(identify->product_id)] = '\0';
7690
7691         memcpy(ctrl_info->vendor, identify->vendor_id,
7692                 sizeof(identify->vendor_id));
7693         ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';
7694
7695         dev_info(&ctrl_info->pci_dev->dev,
7696                 "Firmware version: %s\n", ctrl_info->firmware_version);
7697
7698 out:
7699         kfree(identify);
7700
7701         return rc;
7702 }
7703
7704 struct pqi_config_table_section_info {
7705         struct pqi_ctrl_info *ctrl_info;
7706         void            *section;
7707         u32             section_offset;
7708         void __iomem    *section_iomem_addr;
7709 };
7710
7711 static inline bool pqi_is_firmware_feature_supported(
7712         struct pqi_config_table_firmware_features *firmware_features,
7713         unsigned int bit_position)
7714 {
7715         unsigned int byte_index;
7716
7717         byte_index = bit_position / BITS_PER_BYTE;
7718
7719         if (byte_index >= le16_to_cpu(firmware_features->num_elements))
7720                 return false;
7721
7722         return firmware_features->features_supported[byte_index] &
7723                 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7724 }
7725
7726 static inline bool pqi_is_firmware_feature_enabled(
7727         struct pqi_config_table_firmware_features *firmware_features,
7728         void __iomem *firmware_features_iomem_addr,
7729         unsigned int bit_position)
7730 {
7731         unsigned int byte_index;
7732         u8 __iomem *features_enabled_iomem_addr;
7733
7734         byte_index = (bit_position / BITS_PER_BYTE) +
7735                 (le16_to_cpu(firmware_features->num_elements) * 2);
7736
7737         features_enabled_iomem_addr = firmware_features_iomem_addr +
7738                 offsetof(struct pqi_config_table_firmware_features,
7739                         features_supported) + byte_index;
7740
7741         return *((__force u8 *)features_enabled_iomem_addr) &
7742                 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7743 }
7744
7745 static inline void pqi_request_firmware_feature(
7746         struct pqi_config_table_firmware_features *firmware_features,
7747         unsigned int bit_position)
7748 {
7749         unsigned int byte_index;
7750
7751         byte_index = (bit_position / BITS_PER_BYTE) +
7752                 le16_to_cpu(firmware_features->num_elements);
7753
7754         firmware_features->features_supported[byte_index] |=
7755                 (1 << (bit_position % BITS_PER_BYTE));
7756 }
7757
7758 static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,
7759         u16 first_section, u16 last_section)
7760 {
7761         struct pqi_vendor_general_request request;
7762
7763         memset(&request, 0, sizeof(request));
7764
7765         request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
7766         put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
7767                 &request.header.iu_length);
7768         put_unaligned_le16(PQI_VENDOR_GENERAL_CONFIG_TABLE_UPDATE,
7769                 &request.function_code);
7770         put_unaligned_le16(first_section,
7771                 &request.data.config_table_update.first_section);
7772         put_unaligned_le16(last_section,
7773                 &request.data.config_table_update.last_section);
7774
7775         return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
7776 }
7777
7778 static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,
7779         struct pqi_config_table_firmware_features *firmware_features,
7780         void __iomem *firmware_features_iomem_addr)
7781 {
7782         void *features_requested;
7783         void __iomem *features_requested_iomem_addr;
7784         void __iomem *host_max_known_feature_iomem_addr;
7785
7786         features_requested = firmware_features->features_supported +
7787                 le16_to_cpu(firmware_features->num_elements);
7788
7789         features_requested_iomem_addr = firmware_features_iomem_addr +
7790                 (features_requested - (void *)firmware_features);
7791
7792         memcpy_toio(features_requested_iomem_addr, features_requested,
7793                 le16_to_cpu(firmware_features->num_elements));
7794
7795         if (pqi_is_firmware_feature_supported(firmware_features,
7796                 PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE)) {
7797                 host_max_known_feature_iomem_addr =
7798                         features_requested_iomem_addr +
7799                         (le16_to_cpu(firmware_features->num_elements) * 2) +
7800                         sizeof(__le16);
7801                 writeb(PQI_FIRMWARE_FEATURE_MAXIMUM & 0xFF, host_max_known_feature_iomem_addr);
7802                 writeb((PQI_FIRMWARE_FEATURE_MAXIMUM & 0xFF00) >> 8, host_max_known_feature_iomem_addr + 1);
7803         }
7804
7805         return pqi_config_table_update(ctrl_info,
7806                 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES,
7807                 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES);
7808 }
7809
7810 struct pqi_firmware_feature {
7811         char            *feature_name;
7812         unsigned int    feature_bit;
7813         bool            supported;
7814         bool            enabled;
7815         void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
7816                 struct pqi_firmware_feature *firmware_feature);
7817 };
7818
7819 static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,
7820         struct pqi_firmware_feature *firmware_feature)
7821 {
7822         if (!firmware_feature->supported) {
7823                 dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",
7824                         firmware_feature->feature_name);
7825                 return;
7826         }
7827
7828         if (firmware_feature->enabled) {
7829                 dev_info(&ctrl_info->pci_dev->dev,
7830                         "%s enabled\n", firmware_feature->feature_name);
7831                 return;
7832         }
7833
7834         dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",
7835                 firmware_feature->feature_name);
7836 }
7837
7838 static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
7839         struct pqi_firmware_feature *firmware_feature)
7840 {
7841         switch (firmware_feature->feature_bit) {
7842         case PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS:
7843                 ctrl_info->enable_r1_writes = firmware_feature->enabled;
7844                 break;
7845         case PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS:
7846                 ctrl_info->enable_r5_writes = firmware_feature->enabled;
7847                 break;
7848         case PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS:
7849                 ctrl_info->enable_r6_writes = firmware_feature->enabled;
7850                 break;
7851         case PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE:
7852                 ctrl_info->soft_reset_handshake_supported =
7853                         firmware_feature->enabled &&
7854                         pqi_read_soft_reset_status(ctrl_info);
7855                 break;
7856         case PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT:
7857                 ctrl_info->raid_iu_timeout_supported = firmware_feature->enabled;
7858                 break;
7859         case PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT:
7860                 ctrl_info->tmf_iu_timeout_supported = firmware_feature->enabled;
7861                 break;
7862         case PQI_FIRMWARE_FEATURE_FW_TRIAGE:
7863                 ctrl_info->firmware_triage_supported = firmware_feature->enabled;
7864                 pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
7865                 break;
7866         case PQI_FIRMWARE_FEATURE_RPL_EXTENDED_FORMAT_4_5:
7867                 ctrl_info->rpl_extended_format_4_5_supported = firmware_feature->enabled;
7868                 break;
7869         case PQI_FIRMWARE_FEATURE_MULTI_LUN_DEVICE_SUPPORT:
7870                 ctrl_info->multi_lun_device_supported = firmware_feature->enabled;
7871                 break;
7872         }
7873
7874         pqi_firmware_feature_status(ctrl_info, firmware_feature);
7875 }
7876
7877 static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,
7878         struct pqi_firmware_feature *firmware_feature)
7879 {
7880         if (firmware_feature->feature_status)
7881                 firmware_feature->feature_status(ctrl_info, firmware_feature);
7882 }
7883
7884 static DEFINE_MUTEX(pqi_firmware_features_mutex);
7885
7886 static struct pqi_firmware_feature pqi_firmware_features[] = {
7887         {
7888                 .feature_name = "Online Firmware Activation",
7889                 .feature_bit = PQI_FIRMWARE_FEATURE_OFA,
7890                 .feature_status = pqi_firmware_feature_status,
7891         },
7892         {
7893                 .feature_name = "Serial Management Protocol",
7894                 .feature_bit = PQI_FIRMWARE_FEATURE_SMP,
7895                 .feature_status = pqi_firmware_feature_status,
7896         },
7897         {
7898                 .feature_name = "Maximum Known Feature",
7899                 .feature_bit = PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE,
7900                 .feature_status = pqi_firmware_feature_status,
7901         },
7902         {
7903                 .feature_name = "RAID 0 Read Bypass",
7904                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_READ_BYPASS,
7905                 .feature_status = pqi_firmware_feature_status,
7906         },
7907         {
7908                 .feature_name = "RAID 1 Read Bypass",
7909                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_READ_BYPASS,
7910                 .feature_status = pqi_firmware_feature_status,
7911         },
7912         {
7913                 .feature_name = "RAID 5 Read Bypass",
7914                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_READ_BYPASS,
7915                 .feature_status = pqi_firmware_feature_status,
7916         },
7917         {
7918                 .feature_name = "RAID 6 Read Bypass",
7919                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_READ_BYPASS,
7920                 .feature_status = pqi_firmware_feature_status,
7921         },
7922         {
7923                 .feature_name = "RAID 0 Write Bypass",
7924                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_WRITE_BYPASS,
7925                 .feature_status = pqi_firmware_feature_status,
7926         },
7927         {
7928                 .feature_name = "RAID 1 Write Bypass",
7929                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS,
7930                 .feature_status = pqi_ctrl_update_feature_flags,
7931         },
7932         {
7933                 .feature_name = "RAID 5 Write Bypass",
7934                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS,
7935                 .feature_status = pqi_ctrl_update_feature_flags,
7936         },
7937         {
7938                 .feature_name = "RAID 6 Write Bypass",
7939                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS,
7940                 .feature_status = pqi_ctrl_update_feature_flags,
7941         },
7942         {
7943                 .feature_name = "New Soft Reset Handshake",
7944                 .feature_bit = PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE,
7945                 .feature_status = pqi_ctrl_update_feature_flags,
7946         },
7947         {
7948                 .feature_name = "RAID IU Timeout",
7949                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT,
7950                 .feature_status = pqi_ctrl_update_feature_flags,
7951         },
7952         {
7953                 .feature_name = "TMF IU Timeout",
7954                 .feature_bit = PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT,
7955                 .feature_status = pqi_ctrl_update_feature_flags,
7956         },
7957         {
7958                 .feature_name = "RAID Bypass on encrypted logical volumes on NVMe",
7959                 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_BYPASS_ON_ENCRYPTED_NVME,
7960                 .feature_status = pqi_firmware_feature_status,
7961         },
7962         {
7963                 .feature_name = "Firmware Triage",
7964                 .feature_bit = PQI_FIRMWARE_FEATURE_FW_TRIAGE,
7965                 .feature_status = pqi_ctrl_update_feature_flags,
7966         },
7967         {
7968                 .feature_name = "RPL Extended Formats 4 and 5",
7969                 .feature_bit = PQI_FIRMWARE_FEATURE_RPL_EXTENDED_FORMAT_4_5,
7970                 .feature_status = pqi_ctrl_update_feature_flags,
7971         },
7972         {
7973                 .feature_name = "Multi-LUN Target",
7974                 .feature_bit = PQI_FIRMWARE_FEATURE_MULTI_LUN_DEVICE_SUPPORT,
7975                 .feature_status = pqi_ctrl_update_feature_flags,
7976         },
7977 };
7978
7979 static void pqi_process_firmware_features(
7980         struct pqi_config_table_section_info *section_info)
7981 {
7982         int rc;
7983         struct pqi_ctrl_info *ctrl_info;
7984         struct pqi_config_table_firmware_features *firmware_features;
7985         void __iomem *firmware_features_iomem_addr;
7986         unsigned int i;
7987         unsigned int num_features_supported;
7988
7989         ctrl_info = section_info->ctrl_info;
7990         firmware_features = section_info->section;
7991         firmware_features_iomem_addr = section_info->section_iomem_addr;
7992
7993         for (i = 0, num_features_supported = 0;
7994                 i < ARRAY_SIZE(pqi_firmware_features); i++) {
7995                 if (pqi_is_firmware_feature_supported(firmware_features,
7996                         pqi_firmware_features[i].feature_bit)) {
7997                         pqi_firmware_features[i].supported = true;
7998                         num_features_supported++;
7999                 } else {
8000                         pqi_firmware_feature_update(ctrl_info,
8001                                 &pqi_firmware_features[i]);
8002                 }
8003         }
8004
8005         if (num_features_supported == 0)
8006                 return;
8007
8008         for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8009                 if (!pqi_firmware_features[i].supported)
8010                         continue;
8011                 pqi_request_firmware_feature(firmware_features,
8012                         pqi_firmware_features[i].feature_bit);
8013         }
8014
8015         rc = pqi_enable_firmware_features(ctrl_info, firmware_features,
8016                 firmware_features_iomem_addr);
8017         if (rc) {
8018                 dev_err(&ctrl_info->pci_dev->dev,
8019                         "failed to enable firmware features in PQI configuration table\n");
8020                 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8021                         if (!pqi_firmware_features[i].supported)
8022                                 continue;
8023                         pqi_firmware_feature_update(ctrl_info,
8024                                 &pqi_firmware_features[i]);
8025                 }
8026                 return;
8027         }
8028
8029         for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8030                 if (!pqi_firmware_features[i].supported)
8031                         continue;
8032                 if (pqi_is_firmware_feature_enabled(firmware_features,
8033                         firmware_features_iomem_addr,
8034                         pqi_firmware_features[i].feature_bit)) {
8035                                 pqi_firmware_features[i].enabled = true;
8036                 }
8037                 pqi_firmware_feature_update(ctrl_info,
8038                         &pqi_firmware_features[i]);
8039         }
8040 }
8041
8042 static void pqi_init_firmware_features(void)
8043 {
8044         unsigned int i;
8045
8046         for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8047                 pqi_firmware_features[i].supported = false;
8048                 pqi_firmware_features[i].enabled = false;
8049         }
8050 }
8051
8052 static void pqi_process_firmware_features_section(
8053         struct pqi_config_table_section_info *section_info)
8054 {
8055         mutex_lock(&pqi_firmware_features_mutex);
8056         pqi_init_firmware_features();
8057         pqi_process_firmware_features(section_info);
8058         mutex_unlock(&pqi_firmware_features_mutex);
8059 }
8060
8061 /*
8062  * Reset all controller settings that can be initialized during the processing
8063  * of the PQI Configuration Table.
8064  */
8065
8066 static void pqi_ctrl_reset_config(struct pqi_ctrl_info *ctrl_info)
8067 {
8068         ctrl_info->heartbeat_counter = NULL;
8069         ctrl_info->soft_reset_status = NULL;
8070         ctrl_info->soft_reset_handshake_supported = false;
8071         ctrl_info->enable_r1_writes = false;
8072         ctrl_info->enable_r5_writes = false;
8073         ctrl_info->enable_r6_writes = false;
8074         ctrl_info->raid_iu_timeout_supported = false;
8075         ctrl_info->tmf_iu_timeout_supported = false;
8076         ctrl_info->firmware_triage_supported = false;
8077         ctrl_info->rpl_extended_format_4_5_supported = false;
8078         ctrl_info->multi_lun_device_supported = false;
8079 }
8080
8081 static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)
8082 {
8083         u32 table_length;
8084         u32 section_offset;
8085         bool firmware_feature_section_present;
8086         void __iomem *table_iomem_addr;
8087         struct pqi_config_table *config_table;
8088         struct pqi_config_table_section_header *section;
8089         struct pqi_config_table_section_info section_info;
8090         struct pqi_config_table_section_info feature_section_info = {0};
8091
8092         table_length = ctrl_info->config_table_length;
8093         if (table_length == 0)
8094                 return 0;
8095
8096         config_table = kmalloc(table_length, GFP_KERNEL);
8097         if (!config_table) {
8098                 dev_err(&ctrl_info->pci_dev->dev,
8099                         "failed to allocate memory for PQI configuration table\n");
8100                 return -ENOMEM;
8101         }
8102
8103         /*
8104          * Copy the config table contents from I/O memory space into the
8105          * temporary buffer.
8106          */
8107         table_iomem_addr = ctrl_info->iomem_base + ctrl_info->config_table_offset;
8108         memcpy_fromio(config_table, table_iomem_addr, table_length);
8109
8110         firmware_feature_section_present = false;
8111         section_info.ctrl_info = ctrl_info;
8112         section_offset = get_unaligned_le32(&config_table->first_section_offset);
8113
8114         while (section_offset) {
8115                 section = (void *)config_table + section_offset;
8116
8117                 section_info.section = section;
8118                 section_info.section_offset = section_offset;
8119                 section_info.section_iomem_addr = table_iomem_addr + section_offset;
8120
8121                 switch (get_unaligned_le16(&section->section_id)) {
8122                 case PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES:
8123                         firmware_feature_section_present = true;
8124                         feature_section_info = section_info;
8125                         break;
8126                 case PQI_CONFIG_TABLE_SECTION_HEARTBEAT:
8127                         if (pqi_disable_heartbeat)
8128                                 dev_warn(&ctrl_info->pci_dev->dev,
8129                                 "heartbeat disabled by module parameter\n");
8130                         else
8131                                 ctrl_info->heartbeat_counter =
8132                                         table_iomem_addr +
8133                                         section_offset +
8134                                         offsetof(struct pqi_config_table_heartbeat,
8135                                                 heartbeat_counter);
8136                         break;
8137                 case PQI_CONFIG_TABLE_SECTION_SOFT_RESET:
8138                         ctrl_info->soft_reset_status =
8139                                 table_iomem_addr +
8140                                 section_offset +
8141                                 offsetof(struct pqi_config_table_soft_reset,
8142                                         soft_reset_status);
8143                         break;
8144                 }
8145
8146                 section_offset = get_unaligned_le16(&section->next_section_offset);
8147         }
8148
8149         /*
8150          * We process the firmware feature section after all other sections
8151          * have been processed so that the feature bit callbacks can take
8152          * into account the settings configured by other sections.
8153          */
8154         if (firmware_feature_section_present)
8155                 pqi_process_firmware_features_section(&feature_section_info);
8156
8157         kfree(config_table);
8158
8159         return 0;
8160 }
8161
8162 /* Switches the controller from PQI mode back into SIS mode. */
8163
8164 static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
8165 {
8166         int rc;
8167
8168         pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);
8169         rc = pqi_reset(ctrl_info);
8170         if (rc)
8171                 return rc;
8172         rc = sis_reenable_sis_mode(ctrl_info);
8173         if (rc) {
8174                 dev_err(&ctrl_info->pci_dev->dev,
8175                         "re-enabling SIS mode failed with error %d\n", rc);
8176                 return rc;
8177         }
8178         pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
8179
8180         return 0;
8181 }
8182
8183 /*
8184  * If the controller isn't already in SIS mode, this function forces it into
8185  * SIS mode.
8186  */
8187
8188 static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
8189 {
8190         if (!sis_is_firmware_running(ctrl_info))
8191                 return -ENXIO;
8192
8193         if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)
8194                 return 0;
8195
8196         if (sis_is_kernel_up(ctrl_info)) {
8197                 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
8198                 return 0;
8199         }
8200
8201         return pqi_revert_to_sis_mode(ctrl_info);
8202 }
8203
8204 static void pqi_perform_lockup_action(void)
8205 {
8206         switch (pqi_lockup_action) {
8207         case PANIC:
8208                 panic("FATAL: Smart Family Controller lockup detected");
8209                 break;
8210         case REBOOT:
8211                 emergency_restart();
8212                 break;
8213         case NONE:
8214         default:
8215                 break;
8216         }
8217 }
8218
8219 static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
8220 {
8221         int rc;
8222         u32 product_id;
8223
8224         if (reset_devices) {
8225                 if (pqi_is_fw_triage_supported(ctrl_info)) {
8226                         rc = sis_wait_for_fw_triage_completion(ctrl_info);
8227                         if (rc)
8228                                 return rc;
8229                 }
8230                 sis_soft_reset(ctrl_info);
8231                 ssleep(PQI_POST_RESET_DELAY_SECS);
8232         } else {
8233                 rc = pqi_force_sis_mode(ctrl_info);
8234                 if (rc)
8235                         return rc;
8236         }
8237
8238         /*
8239          * Wait until the controller is ready to start accepting SIS
8240          * commands.
8241          */
8242         rc = sis_wait_for_ctrl_ready(ctrl_info);
8243         if (rc) {
8244                 if (reset_devices) {
8245                         dev_err(&ctrl_info->pci_dev->dev,
8246                                 "kdump init failed with error %d\n", rc);
8247                         pqi_lockup_action = REBOOT;
8248                         pqi_perform_lockup_action();
8249                 }
8250                 return rc;
8251         }
8252
8253         /*
8254          * Get the controller properties.  This allows us to determine
8255          * whether or not it supports PQI mode.
8256          */
8257         rc = sis_get_ctrl_properties(ctrl_info);
8258         if (rc) {
8259                 dev_err(&ctrl_info->pci_dev->dev,
8260                         "error obtaining controller properties\n");
8261                 return rc;
8262         }
8263
8264         rc = sis_get_pqi_capabilities(ctrl_info);
8265         if (rc) {
8266                 dev_err(&ctrl_info->pci_dev->dev,
8267                         "error obtaining controller capabilities\n");
8268                 return rc;
8269         }
8270
8271         product_id = sis_get_product_id(ctrl_info);
8272         ctrl_info->product_id = (u8)product_id;
8273         ctrl_info->product_revision = (u8)(product_id >> 8);
8274
8275         if (reset_devices) {
8276                 if (ctrl_info->max_outstanding_requests >
8277                         PQI_MAX_OUTSTANDING_REQUESTS_KDUMP)
8278                                 ctrl_info->max_outstanding_requests =
8279                                         PQI_MAX_OUTSTANDING_REQUESTS_KDUMP;
8280         } else {
8281                 if (ctrl_info->max_outstanding_requests >
8282                         PQI_MAX_OUTSTANDING_REQUESTS)
8283                                 ctrl_info->max_outstanding_requests =
8284                                         PQI_MAX_OUTSTANDING_REQUESTS;
8285         }
8286
8287         pqi_calculate_io_resources(ctrl_info);
8288
8289         rc = pqi_alloc_error_buffer(ctrl_info);
8290         if (rc) {
8291                 dev_err(&ctrl_info->pci_dev->dev,
8292                         "failed to allocate PQI error buffer\n");
8293                 return rc;
8294         }
8295
8296         /*
8297          * If the function we are about to call succeeds, the
8298          * controller will transition from legacy SIS mode
8299          * into PQI mode.
8300          */
8301         rc = sis_init_base_struct_addr(ctrl_info);
8302         if (rc) {
8303                 dev_err(&ctrl_info->pci_dev->dev,
8304                         "error initializing PQI mode\n");
8305                 return rc;
8306         }
8307
8308         /* Wait for the controller to complete the SIS -> PQI transition. */
8309         rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
8310         if (rc) {
8311                 dev_err(&ctrl_info->pci_dev->dev,
8312                         "transition to PQI mode failed\n");
8313                 return rc;
8314         }
8315
8316         /* From here on, we are running in PQI mode. */
8317         ctrl_info->pqi_mode_enabled = true;
8318         pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
8319
8320         rc = pqi_alloc_admin_queues(ctrl_info);
8321         if (rc) {
8322                 dev_err(&ctrl_info->pci_dev->dev,
8323                         "failed to allocate admin queues\n");
8324                 return rc;
8325         }
8326
8327         rc = pqi_create_admin_queues(ctrl_info);
8328         if (rc) {
8329                 dev_err(&ctrl_info->pci_dev->dev,
8330                         "error creating admin queues\n");
8331                 return rc;
8332         }
8333
8334         rc = pqi_report_device_capability(ctrl_info);
8335         if (rc) {
8336                 dev_err(&ctrl_info->pci_dev->dev,
8337                         "obtaining device capability failed\n");
8338                 return rc;
8339         }
8340
8341         rc = pqi_validate_device_capability(ctrl_info);
8342         if (rc)
8343                 return rc;
8344
8345         pqi_calculate_queue_resources(ctrl_info);
8346
8347         rc = pqi_enable_msix_interrupts(ctrl_info);
8348         if (rc)
8349                 return rc;
8350
8351         if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
8352                 ctrl_info->max_msix_vectors =
8353                         ctrl_info->num_msix_vectors_enabled;
8354                 pqi_calculate_queue_resources(ctrl_info);
8355         }
8356
8357         rc = pqi_alloc_io_resources(ctrl_info);
8358         if (rc)
8359                 return rc;
8360
8361         rc = pqi_alloc_operational_queues(ctrl_info);
8362         if (rc) {
8363                 dev_err(&ctrl_info->pci_dev->dev,
8364                         "failed to allocate operational queues\n");
8365                 return rc;
8366         }
8367
8368         pqi_init_operational_queues(ctrl_info);
8369
8370         rc = pqi_create_queues(ctrl_info);
8371         if (rc)
8372                 return rc;
8373
8374         rc = pqi_request_irqs(ctrl_info);
8375         if (rc)
8376                 return rc;
8377
8378         pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
8379
8380         ctrl_info->controller_online = true;
8381
8382         rc = pqi_process_config_table(ctrl_info);
8383         if (rc)
8384                 return rc;
8385
8386         pqi_start_heartbeat_timer(ctrl_info);
8387
8388         if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
8389                 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
8390                 if (rc) { /* Supported features not returned correctly. */
8391                         dev_err(&ctrl_info->pci_dev->dev,
8392                                 "error obtaining advanced RAID bypass configuration\n");
8393                         return rc;
8394                 }
8395                 ctrl_info->ciss_report_log_flags |=
8396                         CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
8397         }
8398
8399         rc = pqi_enable_events(ctrl_info);
8400         if (rc) {
8401                 dev_err(&ctrl_info->pci_dev->dev,
8402                         "error enabling events\n");
8403                 return rc;
8404         }
8405
8406         /* Register with the SCSI subsystem. */
8407         rc = pqi_register_scsi(ctrl_info);
8408         if (rc)
8409                 return rc;
8410
8411         rc = pqi_get_ctrl_product_details(ctrl_info);
8412         if (rc) {
8413                 dev_err(&ctrl_info->pci_dev->dev,
8414                         "error obtaining product details\n");
8415                 return rc;
8416         }
8417
8418         rc = pqi_get_ctrl_serial_number(ctrl_info);
8419         if (rc) {
8420                 dev_err(&ctrl_info->pci_dev->dev,
8421                         "error obtaining ctrl serial number\n");
8422                 return rc;
8423         }
8424
8425         rc = pqi_set_diag_rescan(ctrl_info);
8426         if (rc) {
8427                 dev_err(&ctrl_info->pci_dev->dev,
8428                         "error enabling multi-lun rescan\n");
8429                 return rc;
8430         }
8431
8432         rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
8433         if (rc) {
8434                 dev_err(&ctrl_info->pci_dev->dev,
8435                         "error updating host wellness\n");
8436                 return rc;
8437         }
8438
8439         pqi_schedule_update_time_worker(ctrl_info);
8440
8441         pqi_scan_scsi_devices(ctrl_info);
8442
8443         return 0;
8444 }
8445
8446 static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)
8447 {
8448         unsigned int i;
8449         struct pqi_admin_queues *admin_queues;
8450         struct pqi_event_queue *event_queue;
8451
8452         admin_queues = &ctrl_info->admin_queues;
8453         admin_queues->iq_pi_copy = 0;
8454         admin_queues->oq_ci_copy = 0;
8455         writel(0, admin_queues->oq_pi);
8456
8457         for (i = 0; i < ctrl_info->num_queue_groups; i++) {
8458                 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;
8459                 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;
8460                 ctrl_info->queue_groups[i].oq_ci_copy = 0;
8461
8462                 writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);
8463                 writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);
8464                 writel(0, ctrl_info->queue_groups[i].oq_pi);
8465         }
8466
8467         event_queue = &ctrl_info->event_queue;
8468         writel(0, event_queue->oq_pi);
8469         event_queue->oq_ci_copy = 0;
8470 }
8471
8472 static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)
8473 {
8474         int rc;
8475
8476         rc = pqi_force_sis_mode(ctrl_info);
8477         if (rc)
8478                 return rc;
8479
8480         /*
8481          * Wait until the controller is ready to start accepting SIS
8482          * commands.
8483          */
8484         rc = sis_wait_for_ctrl_ready_resume(ctrl_info);
8485         if (rc)
8486                 return rc;
8487
8488         /*
8489          * Get the controller properties.  This allows us to determine
8490          * whether or not it supports PQI mode.
8491          */
8492         rc = sis_get_ctrl_properties(ctrl_info);
8493         if (rc) {
8494                 dev_err(&ctrl_info->pci_dev->dev,
8495                         "error obtaining controller properties\n");
8496                 return rc;
8497         }
8498
8499         rc = sis_get_pqi_capabilities(ctrl_info);
8500         if (rc) {
8501                 dev_err(&ctrl_info->pci_dev->dev,
8502                         "error obtaining controller capabilities\n");
8503                 return rc;
8504         }
8505
8506         /*
8507          * If the function we are about to call succeeds, the
8508          * controller will transition from legacy SIS mode
8509          * into PQI mode.
8510          */
8511         rc = sis_init_base_struct_addr(ctrl_info);
8512         if (rc) {
8513                 dev_err(&ctrl_info->pci_dev->dev,
8514                         "error initializing PQI mode\n");
8515                 return rc;
8516         }
8517
8518         /* Wait for the controller to complete the SIS -> PQI transition. */
8519         rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
8520         if (rc) {
8521                 dev_err(&ctrl_info->pci_dev->dev,
8522                         "transition to PQI mode failed\n");
8523                 return rc;
8524         }
8525
8526         /* From here on, we are running in PQI mode. */
8527         ctrl_info->pqi_mode_enabled = true;
8528         pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
8529
8530         pqi_reinit_queues(ctrl_info);
8531
8532         rc = pqi_create_admin_queues(ctrl_info);
8533         if (rc) {
8534                 dev_err(&ctrl_info->pci_dev->dev,
8535                         "error creating admin queues\n");
8536                 return rc;
8537         }
8538
8539         rc = pqi_create_queues(ctrl_info);
8540         if (rc)
8541                 return rc;
8542
8543         pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
8544
8545         ctrl_info->controller_online = true;
8546         pqi_ctrl_unblock_requests(ctrl_info);
8547
8548         pqi_ctrl_reset_config(ctrl_info);
8549
8550         rc = pqi_process_config_table(ctrl_info);
8551         if (rc)
8552                 return rc;
8553
8554         pqi_start_heartbeat_timer(ctrl_info);
8555
8556         if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
8557                 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
8558                 if (rc) {
8559                         dev_err(&ctrl_info->pci_dev->dev,
8560                                 "error obtaining advanced RAID bypass configuration\n");
8561                         return rc;
8562                 }
8563                 ctrl_info->ciss_report_log_flags |=
8564                         CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
8565         }
8566
8567         rc = pqi_enable_events(ctrl_info);
8568         if (rc) {
8569                 dev_err(&ctrl_info->pci_dev->dev,
8570                         "error enabling events\n");
8571                 return rc;
8572         }
8573
8574         rc = pqi_get_ctrl_product_details(ctrl_info);
8575         if (rc) {
8576                 dev_err(&ctrl_info->pci_dev->dev,
8577                         "error obtaining product details\n");
8578                 return rc;
8579         }
8580
8581         rc = pqi_set_diag_rescan(ctrl_info);
8582         if (rc) {
8583                 dev_err(&ctrl_info->pci_dev->dev,
8584                         "error enabling multi-lun rescan\n");
8585                 return rc;
8586         }
8587
8588         rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
8589         if (rc) {
8590                 dev_err(&ctrl_info->pci_dev->dev,
8591                         "error updating host wellness\n");
8592                 return rc;
8593         }
8594
8595         if (pqi_ofa_in_progress(ctrl_info))
8596                 pqi_ctrl_unblock_scan(ctrl_info);
8597
8598         pqi_scan_scsi_devices(ctrl_info);
8599
8600         return 0;
8601 }
8602
8603 static inline int pqi_set_pcie_completion_timeout(struct pci_dev *pci_dev, u16 timeout)
8604 {
8605         int rc;
8606
8607         rc = pcie_capability_clear_and_set_word(pci_dev, PCI_EXP_DEVCTL2,
8608                 PCI_EXP_DEVCTL2_COMP_TIMEOUT, timeout);
8609
8610         return pcibios_err_to_errno(rc);
8611 }
8612
8613 static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
8614 {
8615         int rc;
8616         u64 mask;
8617
8618         rc = pci_enable_device(ctrl_info->pci_dev);
8619         if (rc) {
8620                 dev_err(&ctrl_info->pci_dev->dev,
8621                         "failed to enable PCI device\n");
8622                 return rc;
8623         }
8624
8625         if (sizeof(dma_addr_t) > 4)
8626                 mask = DMA_BIT_MASK(64);
8627         else
8628                 mask = DMA_BIT_MASK(32);
8629
8630         rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);
8631         if (rc) {
8632                 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
8633                 goto disable_device;
8634         }
8635
8636         rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
8637         if (rc) {
8638                 dev_err(&ctrl_info->pci_dev->dev,
8639                         "failed to obtain PCI resources\n");
8640                 goto disable_device;
8641         }
8642
8643         ctrl_info->iomem_base = ioremap(pci_resource_start(
8644                 ctrl_info->pci_dev, 0),
8645                 pci_resource_len(ctrl_info->pci_dev, 0));
8646         if (!ctrl_info->iomem_base) {
8647                 dev_err(&ctrl_info->pci_dev->dev,
8648                         "failed to map memory for controller registers\n");
8649                 rc = -ENOMEM;
8650                 goto release_regions;
8651         }
8652
8653 #define PCI_EXP_COMP_TIMEOUT_65_TO_210_MS               0x6
8654
8655         /* Increase the PCIe completion timeout. */
8656         rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,
8657                 PCI_EXP_COMP_TIMEOUT_65_TO_210_MS);
8658         if (rc) {
8659                 dev_err(&ctrl_info->pci_dev->dev,
8660                         "failed to set PCIe completion timeout\n");
8661                 goto release_regions;
8662         }
8663
8664         /* Enable bus mastering. */
8665         pci_set_master(ctrl_info->pci_dev);
8666
8667         ctrl_info->registers = ctrl_info->iomem_base;
8668         ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
8669
8670         pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
8671
8672         return 0;
8673
8674 release_regions:
8675         pci_release_regions(ctrl_info->pci_dev);
8676 disable_device:
8677         pci_disable_device(ctrl_info->pci_dev);
8678
8679         return rc;
8680 }
8681
8682 static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
8683 {
8684         iounmap(ctrl_info->iomem_base);
8685         pci_release_regions(ctrl_info->pci_dev);
8686         if (pci_is_enabled(ctrl_info->pci_dev))
8687                 pci_disable_device(ctrl_info->pci_dev);
8688         pci_set_drvdata(ctrl_info->pci_dev, NULL);
8689 }
8690
8691 static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
8692 {
8693         struct pqi_ctrl_info *ctrl_info;
8694
8695         ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
8696                         GFP_KERNEL, numa_node);
8697         if (!ctrl_info)
8698                 return NULL;
8699
8700         mutex_init(&ctrl_info->scan_mutex);
8701         mutex_init(&ctrl_info->lun_reset_mutex);
8702         mutex_init(&ctrl_info->ofa_mutex);
8703
8704         INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
8705         spin_lock_init(&ctrl_info->scsi_device_list_lock);
8706
8707         INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
8708         atomic_set(&ctrl_info->num_interrupts, 0);
8709
8710         INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
8711         INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
8712
8713         timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);
8714         INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);
8715
8716         INIT_WORK(&ctrl_info->ofa_memory_alloc_work, pqi_ofa_memory_alloc_worker);
8717         INIT_WORK(&ctrl_info->ofa_quiesce_work, pqi_ofa_quiesce_worker);
8718
8719         sema_init(&ctrl_info->sync_request_sem,
8720                 PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
8721         init_waitqueue_head(&ctrl_info->block_requests_wait);
8722
8723         ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
8724         ctrl_info->irq_mode = IRQ_MODE_NONE;
8725         ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
8726
8727         ctrl_info->ciss_report_log_flags = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID;
8728         ctrl_info->max_transfer_encrypted_sas_sata =
8729                 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_SAS_SATA;
8730         ctrl_info->max_transfer_encrypted_nvme =
8731                 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_NVME;
8732         ctrl_info->max_write_raid_5_6 = PQI_DEFAULT_MAX_WRITE_RAID_5_6;
8733         ctrl_info->max_write_raid_1_10_2drive = ~0;
8734         ctrl_info->max_write_raid_1_10_3drive = ~0;
8735         ctrl_info->disable_managed_interrupts = pqi_disable_managed_interrupts;
8736
8737         return ctrl_info;
8738 }
8739
8740 static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
8741 {
8742         kfree(ctrl_info);
8743 }
8744
8745 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
8746 {
8747         pqi_free_irqs(ctrl_info);
8748         pqi_disable_msix_interrupts(ctrl_info);
8749 }
8750
8751 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
8752 {
8753         pqi_free_interrupts(ctrl_info);
8754         if (ctrl_info->queue_memory_base)
8755                 dma_free_coherent(&ctrl_info->pci_dev->dev,
8756                         ctrl_info->queue_memory_length,
8757                         ctrl_info->queue_memory_base,
8758                         ctrl_info->queue_memory_base_dma_handle);
8759         if (ctrl_info->admin_queue_memory_base)
8760                 dma_free_coherent(&ctrl_info->pci_dev->dev,
8761                         ctrl_info->admin_queue_memory_length,
8762                         ctrl_info->admin_queue_memory_base,
8763                         ctrl_info->admin_queue_memory_base_dma_handle);
8764         pqi_free_all_io_requests(ctrl_info);
8765         if (ctrl_info->error_buffer)
8766                 dma_free_coherent(&ctrl_info->pci_dev->dev,
8767                         ctrl_info->error_buffer_length,
8768                         ctrl_info->error_buffer,
8769                         ctrl_info->error_buffer_dma_handle);
8770         if (ctrl_info->iomem_base)
8771                 pqi_cleanup_pci_init(ctrl_info);
8772         pqi_free_ctrl_info(ctrl_info);
8773 }
8774
8775 static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
8776 {
8777         ctrl_info->controller_online = false;
8778         pqi_stop_heartbeat_timer(ctrl_info);
8779         pqi_ctrl_block_requests(ctrl_info);
8780         pqi_cancel_rescan_worker(ctrl_info);
8781         pqi_cancel_update_time_worker(ctrl_info);
8782         if (ctrl_info->ctrl_removal_state == PQI_CTRL_SURPRISE_REMOVAL) {
8783                 pqi_fail_all_outstanding_requests(ctrl_info);
8784                 ctrl_info->pqi_mode_enabled = false;
8785         }
8786         pqi_unregister_scsi(ctrl_info);
8787         if (ctrl_info->pqi_mode_enabled)
8788                 pqi_revert_to_sis_mode(ctrl_info);
8789         pqi_free_ctrl_resources(ctrl_info);
8790 }
8791
8792 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)
8793 {
8794         pqi_ctrl_block_scan(ctrl_info);
8795         pqi_scsi_block_requests(ctrl_info);
8796         pqi_ctrl_block_device_reset(ctrl_info);
8797         pqi_ctrl_block_requests(ctrl_info);
8798         pqi_ctrl_wait_until_quiesced(ctrl_info);
8799         pqi_stop_heartbeat_timer(ctrl_info);
8800 }
8801
8802 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)
8803 {
8804         pqi_start_heartbeat_timer(ctrl_info);
8805         pqi_ctrl_unblock_requests(ctrl_info);
8806         pqi_ctrl_unblock_device_reset(ctrl_info);
8807         pqi_scsi_unblock_requests(ctrl_info);
8808         pqi_ctrl_unblock_scan(ctrl_info);
8809 }
8810
8811 static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info, u32 total_size, u32 chunk_size)
8812 {
8813         int i;
8814         u32 sg_count;
8815         struct device *dev;
8816         struct pqi_ofa_memory *ofap;
8817         struct pqi_sg_descriptor *mem_descriptor;
8818         dma_addr_t dma_handle;
8819
8820         ofap = ctrl_info->pqi_ofa_mem_virt_addr;
8821
8822         sg_count = DIV_ROUND_UP(total_size, chunk_size);
8823         if (sg_count == 0 || sg_count > PQI_OFA_MAX_SG_DESCRIPTORS)
8824                 goto out;
8825
8826         ctrl_info->pqi_ofa_chunk_virt_addr = kmalloc_array(sg_count, sizeof(void *), GFP_KERNEL);
8827         if (!ctrl_info->pqi_ofa_chunk_virt_addr)
8828                 goto out;
8829
8830         dev = &ctrl_info->pci_dev->dev;
8831
8832         for (i = 0; i < sg_count; i++) {
8833                 ctrl_info->pqi_ofa_chunk_virt_addr[i] =
8834                         dma_alloc_coherent(dev, chunk_size, &dma_handle, GFP_KERNEL);
8835                 if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])
8836                         goto out_free_chunks;
8837                 mem_descriptor = &ofap->sg_descriptor[i];
8838                 put_unaligned_le64((u64)dma_handle, &mem_descriptor->address);
8839                 put_unaligned_le32(chunk_size, &mem_descriptor->length);
8840         }
8841
8842         put_unaligned_le32(CISS_SG_LAST, &mem_descriptor->flags);
8843         put_unaligned_le16(sg_count, &ofap->num_memory_descriptors);
8844         put_unaligned_le32(sg_count * chunk_size, &ofap->bytes_allocated);
8845
8846         return 0;
8847
8848 out_free_chunks:
8849         while (--i >= 0) {
8850                 mem_descriptor = &ofap->sg_descriptor[i];
8851                 dma_free_coherent(dev, chunk_size,
8852                         ctrl_info->pqi_ofa_chunk_virt_addr[i],
8853                         get_unaligned_le64(&mem_descriptor->address));
8854         }
8855         kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
8856
8857 out:
8858         return -ENOMEM;
8859 }
8860
8861 static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)
8862 {
8863         u32 total_size;
8864         u32 chunk_size;
8865         u32 min_chunk_size;
8866
8867         if (ctrl_info->ofa_bytes_requested == 0)
8868                 return 0;
8869
8870         total_size = PAGE_ALIGN(ctrl_info->ofa_bytes_requested);
8871         min_chunk_size = DIV_ROUND_UP(total_size, PQI_OFA_MAX_SG_DESCRIPTORS);
8872         min_chunk_size = PAGE_ALIGN(min_chunk_size);
8873
8874         for (chunk_size = total_size; chunk_size >= min_chunk_size;) {
8875                 if (pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_size) == 0)
8876                         return 0;
8877                 chunk_size /= 2;
8878                 chunk_size = PAGE_ALIGN(chunk_size);
8879         }
8880
8881         return -ENOMEM;
8882 }
8883
8884 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info)
8885 {
8886         struct device *dev;
8887         struct pqi_ofa_memory *ofap;
8888
8889         dev = &ctrl_info->pci_dev->dev;
8890
8891         ofap = dma_alloc_coherent(dev, sizeof(*ofap),
8892                 &ctrl_info->pqi_ofa_mem_dma_handle, GFP_KERNEL);
8893         if (!ofap)
8894                 return;
8895
8896         ctrl_info->pqi_ofa_mem_virt_addr = ofap;
8897
8898         if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {
8899                 dev_err(dev,
8900                         "failed to allocate host buffer for Online Firmware Activation\n");
8901                 dma_free_coherent(dev, sizeof(*ofap), ofap, ctrl_info->pqi_ofa_mem_dma_handle);
8902                 ctrl_info->pqi_ofa_mem_virt_addr = NULL;
8903                 return;
8904         }
8905
8906         put_unaligned_le16(PQI_OFA_VERSION, &ofap->version);
8907         memcpy(&ofap->signature, PQI_OFA_SIGNATURE, sizeof(ofap->signature));
8908 }
8909
8910 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)
8911 {
8912         unsigned int i;
8913         struct device *dev;
8914         struct pqi_ofa_memory *ofap;
8915         struct pqi_sg_descriptor *mem_descriptor;
8916         unsigned int num_memory_descriptors;
8917
8918         ofap = ctrl_info->pqi_ofa_mem_virt_addr;
8919         if (!ofap)
8920                 return;
8921
8922         dev = &ctrl_info->pci_dev->dev;
8923
8924         if (get_unaligned_le32(&ofap->bytes_allocated) == 0)
8925                 goto out;
8926
8927         mem_descriptor = ofap->sg_descriptor;
8928         num_memory_descriptors =
8929                 get_unaligned_le16(&ofap->num_memory_descriptors);
8930
8931         for (i = 0; i < num_memory_descriptors; i++) {
8932                 dma_free_coherent(dev,
8933                         get_unaligned_le32(&mem_descriptor[i].length),
8934                         ctrl_info->pqi_ofa_chunk_virt_addr[i],
8935                         get_unaligned_le64(&mem_descriptor[i].address));
8936         }
8937         kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
8938
8939 out:
8940         dma_free_coherent(dev, sizeof(*ofap), ofap,
8941                 ctrl_info->pqi_ofa_mem_dma_handle);
8942         ctrl_info->pqi_ofa_mem_virt_addr = NULL;
8943 }
8944
8945 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)
8946 {
8947         u32 buffer_length;
8948         struct pqi_vendor_general_request request;
8949         struct pqi_ofa_memory *ofap;
8950
8951         memset(&request, 0, sizeof(request));
8952
8953         request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
8954         put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
8955                 &request.header.iu_length);
8956         put_unaligned_le16(PQI_VENDOR_GENERAL_HOST_MEMORY_UPDATE,
8957                 &request.function_code);
8958
8959         ofap = ctrl_info->pqi_ofa_mem_virt_addr;
8960
8961         if (ofap) {
8962                 buffer_length = offsetof(struct pqi_ofa_memory, sg_descriptor) +
8963                         get_unaligned_le16(&ofap->num_memory_descriptors) *
8964                         sizeof(struct pqi_sg_descriptor);
8965
8966                 put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,
8967                         &request.data.ofa_memory_allocation.buffer_address);
8968                 put_unaligned_le32(buffer_length,
8969                         &request.data.ofa_memory_allocation.buffer_length);
8970         }
8971
8972         return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
8973 }
8974
8975 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs)
8976 {
8977         ssleep(delay_secs);
8978
8979         return pqi_ctrl_init_resume(ctrl_info);
8980 }
8981
8982 static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
8983         .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
8984         .status = SAM_STAT_CHECK_CONDITION,
8985 };
8986
8987 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)
8988 {
8989         unsigned int i;
8990         struct pqi_io_request *io_request;
8991         struct scsi_cmnd *scmd;
8992         struct scsi_device *sdev;
8993
8994         for (i = 0; i < ctrl_info->max_io_slots; i++) {
8995                 io_request = &ctrl_info->io_request_pool[i];
8996                 if (atomic_read(&io_request->refcount) == 0)
8997                         continue;
8998
8999                 scmd = io_request->scmd;
9000                 if (scmd) {
9001                         sdev = scmd->device;
9002                         if (!sdev || !scsi_device_online(sdev)) {
9003                                 pqi_free_io_request(io_request);
9004                                 continue;
9005                         } else {
9006                                 set_host_byte(scmd, DID_NO_CONNECT);
9007                         }
9008                 } else {
9009                         io_request->status = -ENXIO;
9010                         io_request->error_info =
9011                                 &pqi_ctrl_offline_raid_error_info;
9012                 }
9013
9014                 io_request->io_complete_callback(io_request,
9015                         io_request->context);
9016         }
9017 }
9018
9019 static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)
9020 {
9021         pqi_perform_lockup_action();
9022         pqi_stop_heartbeat_timer(ctrl_info);
9023         pqi_free_interrupts(ctrl_info);
9024         pqi_cancel_rescan_worker(ctrl_info);
9025         pqi_cancel_update_time_worker(ctrl_info);
9026         pqi_ctrl_wait_until_quiesced(ctrl_info);
9027         pqi_fail_all_outstanding_requests(ctrl_info);
9028         pqi_ctrl_unblock_requests(ctrl_info);
9029 }
9030
9031 static void pqi_ctrl_offline_worker(struct work_struct *work)
9032 {
9033         struct pqi_ctrl_info *ctrl_info;
9034
9035         ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);
9036         pqi_take_ctrl_offline_deferred(ctrl_info);
9037 }
9038
9039 static char *pqi_ctrl_shutdown_reason_to_string(enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
9040 {
9041         char *string;
9042
9043         switch (ctrl_shutdown_reason) {
9044         case PQI_IQ_NOT_DRAINED_TIMEOUT:
9045                 string = "inbound queue not drained timeout";
9046                 break;
9047         case PQI_LUN_RESET_TIMEOUT:
9048                 string = "LUN reset timeout";
9049                 break;
9050         case PQI_IO_PENDING_POST_LUN_RESET_TIMEOUT:
9051                 string = "I/O pending timeout after LUN reset";
9052                 break;
9053         case PQI_NO_HEARTBEAT:
9054                 string = "no controller heartbeat detected";
9055                 break;
9056         case PQI_FIRMWARE_KERNEL_NOT_UP:
9057                 string = "firmware kernel not ready";
9058                 break;
9059         case PQI_OFA_RESPONSE_TIMEOUT:
9060                 string = "OFA response timeout";
9061                 break;
9062         case PQI_INVALID_REQ_ID:
9063                 string = "invalid request ID";
9064                 break;
9065         case PQI_UNMATCHED_REQ_ID:
9066                 string = "unmatched request ID";
9067                 break;
9068         case PQI_IO_PI_OUT_OF_RANGE:
9069                 string = "I/O queue producer index out of range";
9070                 break;
9071         case PQI_EVENT_PI_OUT_OF_RANGE:
9072                 string = "event queue producer index out of range";
9073                 break;
9074         case PQI_UNEXPECTED_IU_TYPE:
9075                 string = "unexpected IU type";
9076                 break;
9077         default:
9078                 string = "unknown reason";
9079                 break;
9080         }
9081
9082         return string;
9083 }
9084
9085 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,
9086         enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
9087 {
9088         if (!ctrl_info->controller_online)
9089                 return;
9090
9091         ctrl_info->controller_online = false;
9092         ctrl_info->pqi_mode_enabled = false;
9093         pqi_ctrl_block_requests(ctrl_info);
9094         if (!pqi_disable_ctrl_shutdown)
9095                 sis_shutdown_ctrl(ctrl_info, ctrl_shutdown_reason);
9096         pci_disable_device(ctrl_info->pci_dev);
9097         dev_err(&ctrl_info->pci_dev->dev,
9098                 "controller offline: reason code 0x%x (%s)\n",
9099                 ctrl_shutdown_reason, pqi_ctrl_shutdown_reason_to_string(ctrl_shutdown_reason));
9100         schedule_work(&ctrl_info->ctrl_offline_work);
9101 }
9102
9103 static void pqi_print_ctrl_info(struct pci_dev *pci_dev,
9104         const struct pci_device_id *id)
9105 {
9106         char *ctrl_description;
9107
9108         if (id->driver_data)
9109                 ctrl_description = (char *)id->driver_data;
9110         else
9111                 ctrl_description = "Microchip Smart Family Controller";
9112
9113         dev_info(&pci_dev->dev, "%s found\n", ctrl_description);
9114 }
9115
9116 static int pqi_pci_probe(struct pci_dev *pci_dev,
9117         const struct pci_device_id *id)
9118 {
9119         int rc;
9120         int node;
9121         struct pqi_ctrl_info *ctrl_info;
9122
9123         pqi_print_ctrl_info(pci_dev, id);
9124
9125         if (pqi_disable_device_id_wildcards &&
9126                 id->subvendor == PCI_ANY_ID &&
9127                 id->subdevice == PCI_ANY_ID) {
9128                 dev_warn(&pci_dev->dev,
9129                         "controller not probed because device ID wildcards are disabled\n");
9130                 return -ENODEV;
9131         }
9132
9133         if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
9134                 dev_warn(&pci_dev->dev,
9135                         "controller device ID matched using wildcards\n");
9136
9137         node = dev_to_node(&pci_dev->dev);
9138         if (node == NUMA_NO_NODE) {
9139                 node = cpu_to_node(0);
9140                 if (node == NUMA_NO_NODE)
9141                         node = 0;
9142                 set_dev_node(&pci_dev->dev, node);
9143         }
9144
9145         ctrl_info = pqi_alloc_ctrl_info(node);
9146         if (!ctrl_info) {
9147                 dev_err(&pci_dev->dev,
9148                         "failed to allocate controller info block\n");
9149                 return -ENOMEM;
9150         }
9151         ctrl_info->numa_node = node;
9152
9153         ctrl_info->pci_dev = pci_dev;
9154
9155         rc = pqi_pci_init(ctrl_info);
9156         if (rc)
9157                 goto error;
9158
9159         rc = pqi_ctrl_init(ctrl_info);
9160         if (rc)
9161                 goto error;
9162
9163         return 0;
9164
9165 error:
9166         pqi_remove_ctrl(ctrl_info);
9167
9168         return rc;
9169 }
9170
9171 static void pqi_pci_remove(struct pci_dev *pci_dev)
9172 {
9173         struct pqi_ctrl_info *ctrl_info;
9174         u16 vendor_id;
9175         int rc;
9176
9177         ctrl_info = pci_get_drvdata(pci_dev);
9178         if (!ctrl_info)
9179                 return;
9180
9181         pci_read_config_word(ctrl_info->pci_dev, PCI_SUBSYSTEM_VENDOR_ID, &vendor_id);
9182         if (vendor_id == 0xffff)
9183                 ctrl_info->ctrl_removal_state = PQI_CTRL_SURPRISE_REMOVAL;
9184         else
9185                 ctrl_info->ctrl_removal_state = PQI_CTRL_GRACEFUL_REMOVAL;
9186
9187         if (ctrl_info->ctrl_removal_state == PQI_CTRL_GRACEFUL_REMOVAL) {
9188                 rc = pqi_flush_cache(ctrl_info, RESTART);
9189                 if (rc)
9190                         dev_err(&pci_dev->dev,
9191                                 "unable to flush controller cache during remove\n");
9192         }
9193
9194         pqi_remove_ctrl(ctrl_info);
9195 }
9196
9197 static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)
9198 {
9199         unsigned int i;
9200         struct pqi_io_request *io_request;
9201         struct scsi_cmnd *scmd;
9202
9203         for (i = 0; i < ctrl_info->max_io_slots; i++) {
9204                 io_request = &ctrl_info->io_request_pool[i];
9205                 if (atomic_read(&io_request->refcount) == 0)
9206                         continue;
9207                 scmd = io_request->scmd;
9208                 WARN_ON(scmd != NULL); /* IO command from SML */
9209                 WARN_ON(scmd == NULL); /* Non-IO cmd or driver initiated*/
9210         }
9211 }
9212
9213 static void pqi_shutdown(struct pci_dev *pci_dev)
9214 {
9215         int rc;
9216         struct pqi_ctrl_info *ctrl_info;
9217         enum bmic_flush_cache_shutdown_event shutdown_event;
9218
9219         ctrl_info = pci_get_drvdata(pci_dev);
9220         if (!ctrl_info) {
9221                 dev_err(&pci_dev->dev,
9222                         "cache could not be flushed\n");
9223                 return;
9224         }
9225
9226         pqi_wait_until_ofa_finished(ctrl_info);
9227
9228         pqi_scsi_block_requests(ctrl_info);
9229         pqi_ctrl_block_device_reset(ctrl_info);
9230         pqi_ctrl_block_requests(ctrl_info);
9231         pqi_ctrl_wait_until_quiesced(ctrl_info);
9232
9233         if (system_state == SYSTEM_RESTART)
9234                 shutdown_event = RESTART;
9235         else
9236                 shutdown_event = SHUTDOWN;
9237
9238         /*
9239          * Write all data in the controller's battery-backed cache to
9240          * storage.
9241          */
9242         rc = pqi_flush_cache(ctrl_info, shutdown_event);
9243         if (rc)
9244                 dev_err(&pci_dev->dev,
9245                         "unable to flush controller cache during shutdown\n");
9246
9247         pqi_crash_if_pending_command(ctrl_info);
9248         pqi_reset(ctrl_info);
9249 }
9250
9251 static void pqi_process_lockup_action_param(void)
9252 {
9253         unsigned int i;
9254
9255         if (!pqi_lockup_action_param)
9256                 return;
9257
9258         for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
9259                 if (strcmp(pqi_lockup_action_param,
9260                         pqi_lockup_actions[i].name) == 0) {
9261                         pqi_lockup_action = pqi_lockup_actions[i].action;
9262                         return;
9263                 }
9264         }
9265
9266         pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n",
9267                 DRIVER_NAME_SHORT, pqi_lockup_action_param);
9268 }
9269
9270 #define PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS           30
9271 #define PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS           (30 * 60)
9272
9273 static void pqi_process_ctrl_ready_timeout_param(void)
9274 {
9275         if (pqi_ctrl_ready_timeout_secs == 0)
9276                 return;
9277
9278         if (pqi_ctrl_ready_timeout_secs < PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS) {
9279                 pr_warn("%s: ctrl_ready_timeout parm of %u second(s) is less than minimum timeout of %d seconds - setting timeout to %d seconds\n",
9280                         DRIVER_NAME_SHORT, pqi_ctrl_ready_timeout_secs, PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS, PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS);
9281                 pqi_ctrl_ready_timeout_secs = PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS;
9282         } else if (pqi_ctrl_ready_timeout_secs > PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS) {
9283                 pr_warn("%s: ctrl_ready_timeout parm of %u seconds is greater than maximum timeout of %d seconds - setting timeout to %d seconds\n",
9284                         DRIVER_NAME_SHORT, pqi_ctrl_ready_timeout_secs, PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS, PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS);
9285                 pqi_ctrl_ready_timeout_secs = PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS;
9286         }
9287
9288         sis_ctrl_ready_timeout_secs = pqi_ctrl_ready_timeout_secs;
9289 }
9290
9291 static void pqi_process_module_params(void)
9292 {
9293         pqi_process_lockup_action_param();
9294         pqi_process_ctrl_ready_timeout_param();
9295 }
9296
9297 #if defined(CONFIG_PM)
9298
9299 static inline enum bmic_flush_cache_shutdown_event pqi_get_flush_cache_shutdown_event(struct pci_dev *pci_dev)
9300 {
9301         if (pci_dev->subsystem_vendor == PCI_VENDOR_ID_ADAPTEC2 && pci_dev->subsystem_device == 0x1304)
9302                 return RESTART;
9303
9304         return SUSPEND;
9305 }
9306
9307 static int pqi_suspend_or_freeze(struct device *dev, bool suspend)
9308 {
9309         struct pci_dev *pci_dev;
9310         struct pqi_ctrl_info *ctrl_info;
9311
9312         pci_dev = to_pci_dev(dev);
9313         ctrl_info = pci_get_drvdata(pci_dev);
9314
9315         pqi_wait_until_ofa_finished(ctrl_info);
9316
9317         pqi_ctrl_block_scan(ctrl_info);
9318         pqi_scsi_block_requests(ctrl_info);
9319         pqi_ctrl_block_device_reset(ctrl_info);
9320         pqi_ctrl_block_requests(ctrl_info);
9321         pqi_ctrl_wait_until_quiesced(ctrl_info);
9322
9323         if (suspend) {
9324                 enum bmic_flush_cache_shutdown_event shutdown_event;
9325
9326                 shutdown_event = pqi_get_flush_cache_shutdown_event(pci_dev);
9327                 pqi_flush_cache(ctrl_info, shutdown_event);
9328         }
9329
9330         pqi_stop_heartbeat_timer(ctrl_info);
9331         pqi_crash_if_pending_command(ctrl_info);
9332         pqi_free_irqs(ctrl_info);
9333
9334         ctrl_info->controller_online = false;
9335         ctrl_info->pqi_mode_enabled = false;
9336
9337         return 0;
9338 }
9339
9340 static __maybe_unused int pqi_suspend(struct device *dev)
9341 {
9342         return pqi_suspend_or_freeze(dev, true);
9343 }
9344
9345 static int pqi_resume_or_restore(struct device *dev)
9346 {
9347         int rc;
9348         struct pci_dev *pci_dev;
9349         struct pqi_ctrl_info *ctrl_info;
9350
9351         pci_dev = to_pci_dev(dev);
9352         ctrl_info = pci_get_drvdata(pci_dev);
9353
9354         rc = pqi_request_irqs(ctrl_info);
9355         if (rc)
9356                 return rc;
9357
9358         pqi_ctrl_unblock_device_reset(ctrl_info);
9359         pqi_ctrl_unblock_requests(ctrl_info);
9360         pqi_scsi_unblock_requests(ctrl_info);
9361         pqi_ctrl_unblock_scan(ctrl_info);
9362
9363         ssleep(PQI_POST_RESET_DELAY_SECS);
9364
9365         return pqi_ctrl_init_resume(ctrl_info);
9366 }
9367
9368 static int pqi_freeze(struct device *dev)
9369 {
9370         return pqi_suspend_or_freeze(dev, false);
9371 }
9372
9373 static int pqi_thaw(struct device *dev)
9374 {
9375         int rc;
9376         struct pci_dev *pci_dev;
9377         struct pqi_ctrl_info *ctrl_info;
9378
9379         pci_dev = to_pci_dev(dev);
9380         ctrl_info = pci_get_drvdata(pci_dev);
9381
9382         rc = pqi_request_irqs(ctrl_info);
9383         if (rc)
9384                 return rc;
9385
9386         ctrl_info->controller_online = true;
9387         ctrl_info->pqi_mode_enabled = true;
9388
9389         pqi_ctrl_unblock_device_reset(ctrl_info);
9390         pqi_ctrl_unblock_requests(ctrl_info);
9391         pqi_scsi_unblock_requests(ctrl_info);
9392         pqi_ctrl_unblock_scan(ctrl_info);
9393
9394         return 0;
9395 }
9396
9397 static int pqi_poweroff(struct device *dev)
9398 {
9399         struct pci_dev *pci_dev;
9400         struct pqi_ctrl_info *ctrl_info;
9401         enum bmic_flush_cache_shutdown_event shutdown_event;
9402
9403         pci_dev = to_pci_dev(dev);
9404         ctrl_info = pci_get_drvdata(pci_dev);
9405
9406         shutdown_event = pqi_get_flush_cache_shutdown_event(pci_dev);
9407         pqi_flush_cache(ctrl_info, shutdown_event);
9408
9409         return 0;
9410 }
9411
9412 static const struct dev_pm_ops pqi_pm_ops = {
9413         .suspend = pqi_suspend,
9414         .resume = pqi_resume_or_restore,
9415         .freeze = pqi_freeze,
9416         .thaw = pqi_thaw,
9417         .poweroff = pqi_poweroff,
9418         .restore = pqi_resume_or_restore,
9419 };
9420
9421 #endif /* CONFIG_PM */
9422
9423 /* Define the PCI IDs for the controllers that we support. */
9424 static const struct pci_device_id pqi_pci_id_table[] = {
9425         {
9426                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9427                                0x105b, 0x1211)
9428         },
9429         {
9430                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9431                                0x105b, 0x1321)
9432         },
9433         {
9434                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9435                                0x152d, 0x8a22)
9436         },
9437         {
9438                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9439                                0x152d, 0x8a23)
9440         },
9441         {
9442                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9443                                0x152d, 0x8a24)
9444         },
9445         {
9446                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9447                                0x152d, 0x8a36)
9448         },
9449         {
9450                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9451                                0x152d, 0x8a37)
9452         },
9453         {
9454                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9455                                0x193d, 0x1104)
9456         },
9457         {
9458                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9459                                0x193d, 0x1105)
9460         },
9461         {
9462                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9463                                0x193d, 0x1106)
9464         },
9465         {
9466                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9467                                0x193d, 0x1107)
9468         },
9469         {
9470                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9471                                0x193d, 0x1108)
9472         },
9473         {
9474                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9475                                0x193d, 0x1109)
9476         },
9477         {
9478                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9479                                0x193d, 0x110b)
9480         },
9481         {
9482                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9483                                0x193d, 0x8460)
9484         },
9485         {
9486                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9487                                0x193d, 0x8461)
9488         },
9489         {
9490                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9491                                0x193d, 0xc460)
9492         },
9493         {
9494                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9495                                0x193d, 0xc461)
9496         },
9497         {
9498                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9499                                0x193d, 0xf460)
9500         },
9501         {
9502                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9503                                0x193d, 0xf461)
9504         },
9505         {
9506                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9507                                0x1bd4, 0x0045)
9508         },
9509         {
9510                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9511                                0x1bd4, 0x0046)
9512         },
9513         {
9514                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9515                                0x1bd4, 0x0047)
9516         },
9517         {
9518                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9519                                0x1bd4, 0x0048)
9520         },
9521         {
9522                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9523                                0x1bd4, 0x004a)
9524         },
9525         {
9526                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9527                                0x1bd4, 0x004b)
9528         },
9529         {
9530                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9531                                0x1bd4, 0x004c)
9532         },
9533         {
9534                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9535                                0x1bd4, 0x004f)
9536         },
9537         {
9538                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9539                                0x1bd4, 0x0051)
9540         },
9541         {
9542                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9543                                0x1bd4, 0x0052)
9544         },
9545         {
9546                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9547                                0x1bd4, 0x0053)
9548         },
9549         {
9550                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9551                                0x1bd4, 0x0054)
9552         },
9553         {
9554                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9555                                0x1bd4, 0x006b)
9556         },
9557         {
9558                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9559                                0x1bd4, 0x006c)
9560         },
9561         {
9562                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9563                                0x1bd4, 0x006d)
9564         },
9565         {
9566                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9567                                0x1bd4, 0x006f)
9568         },
9569         {
9570                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9571                                0x1bd4, 0x0070)
9572         },
9573         {
9574                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9575                                0x1bd4, 0x0071)
9576         },
9577         {
9578                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9579                                0x1bd4, 0x0072)
9580         },
9581         {
9582                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9583                                0x1bd4, 0x0086)
9584         },
9585         {
9586                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9587                                0x1bd4, 0x0087)
9588         },
9589         {
9590                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9591                                0x1bd4, 0x0088)
9592         },
9593         {
9594                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9595                                0x1bd4, 0x0089)
9596         },
9597         {
9598                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9599                                0x19e5, 0xd227)
9600         },
9601         {
9602                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9603                                0x19e5, 0xd228)
9604         },
9605         {
9606                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9607                                0x19e5, 0xd229)
9608         },
9609         {
9610                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9611                                0x19e5, 0xd22a)
9612         },
9613         {
9614                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9615                                0x19e5, 0xd22b)
9616         },
9617         {
9618                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9619                                0x19e5, 0xd22c)
9620         },
9621         {
9622                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9623                                PCI_VENDOR_ID_ADAPTEC2, 0x0110)
9624         },
9625         {
9626                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9627                                PCI_VENDOR_ID_ADAPTEC2, 0x0608)
9628         },
9629         {
9630                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9631                                PCI_VENDOR_ID_ADAPTEC2, 0x0659)
9632         },
9633         {
9634                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9635                                PCI_VENDOR_ID_ADAPTEC2, 0x0800)
9636         },
9637         {
9638                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9639                                PCI_VENDOR_ID_ADAPTEC2, 0x0801)
9640         },
9641         {
9642                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9643                                PCI_VENDOR_ID_ADAPTEC2, 0x0802)
9644         },
9645         {
9646                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9647                                PCI_VENDOR_ID_ADAPTEC2, 0x0803)
9648         },
9649         {
9650                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9651                                PCI_VENDOR_ID_ADAPTEC2, 0x0804)
9652         },
9653         {
9654                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9655                                PCI_VENDOR_ID_ADAPTEC2, 0x0805)
9656         },
9657         {
9658                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9659                                PCI_VENDOR_ID_ADAPTEC2, 0x0806)
9660         },
9661         {
9662                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9663                                PCI_VENDOR_ID_ADAPTEC2, 0x0807)
9664         },
9665         {
9666                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9667                                PCI_VENDOR_ID_ADAPTEC2, 0x0808)
9668         },
9669         {
9670                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9671                                PCI_VENDOR_ID_ADAPTEC2, 0x0809)
9672         },
9673         {
9674                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9675                                PCI_VENDOR_ID_ADAPTEC2, 0x080a)
9676         },
9677         {
9678                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9679                                PCI_VENDOR_ID_ADAPTEC2, 0x0900)
9680         },
9681         {
9682                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9683                                PCI_VENDOR_ID_ADAPTEC2, 0x0901)
9684         },
9685         {
9686                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9687                                PCI_VENDOR_ID_ADAPTEC2, 0x0902)
9688         },
9689         {
9690                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9691                                PCI_VENDOR_ID_ADAPTEC2, 0x0903)
9692         },
9693         {
9694                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9695                                PCI_VENDOR_ID_ADAPTEC2, 0x0904)
9696         },
9697         {
9698                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9699                                PCI_VENDOR_ID_ADAPTEC2, 0x0905)
9700         },
9701         {
9702                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9703                                PCI_VENDOR_ID_ADAPTEC2, 0x0906)
9704         },
9705         {
9706                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9707                                PCI_VENDOR_ID_ADAPTEC2, 0x0907)
9708         },
9709         {
9710                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9711                                PCI_VENDOR_ID_ADAPTEC2, 0x0908)
9712         },
9713         {
9714                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9715                                PCI_VENDOR_ID_ADAPTEC2, 0x090a)
9716         },
9717         {
9718                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9719                                PCI_VENDOR_ID_ADAPTEC2, 0x1200)
9720         },
9721         {
9722                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9723                                PCI_VENDOR_ID_ADAPTEC2, 0x1201)
9724         },
9725         {
9726                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9727                                PCI_VENDOR_ID_ADAPTEC2, 0x1202)
9728         },
9729         {
9730                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9731                                PCI_VENDOR_ID_ADAPTEC2, 0x1280)
9732         },
9733         {
9734                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9735                                PCI_VENDOR_ID_ADAPTEC2, 0x1281)
9736         },
9737         {
9738                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9739                                PCI_VENDOR_ID_ADAPTEC2, 0x1282)
9740         },
9741         {
9742                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9743                                PCI_VENDOR_ID_ADAPTEC2, 0x1300)
9744         },
9745         {
9746                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9747                                PCI_VENDOR_ID_ADAPTEC2, 0x1301)
9748         },
9749         {
9750                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9751                                PCI_VENDOR_ID_ADAPTEC2, 0x1302)
9752         },
9753         {
9754                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9755                                PCI_VENDOR_ID_ADAPTEC2, 0x1303)
9756         },
9757         {
9758                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9759                                PCI_VENDOR_ID_ADAPTEC2, 0x1304)
9760         },
9761         {
9762                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9763                                PCI_VENDOR_ID_ADAPTEC2, 0x1380)
9764         },
9765         {
9766                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9767                                PCI_VENDOR_ID_ADAPTEC2, 0x1400)
9768         },
9769         {
9770                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9771                                PCI_VENDOR_ID_ADAPTEC2, 0x1402)
9772         },
9773         {
9774                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9775                                PCI_VENDOR_ID_ADAPTEC2, 0x1410)
9776         },
9777         {
9778                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9779                                PCI_VENDOR_ID_ADAPTEC2, 0x1411)
9780         },
9781         {
9782                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9783                                PCI_VENDOR_ID_ADAPTEC2, 0x1412)
9784         },
9785         {
9786                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9787                                PCI_VENDOR_ID_ADAPTEC2, 0x1420)
9788         },
9789         {
9790                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9791                                PCI_VENDOR_ID_ADAPTEC2, 0x1430)
9792         },
9793         {
9794                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9795                                PCI_VENDOR_ID_ADAPTEC2, 0x1440)
9796         },
9797         {
9798                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9799                                PCI_VENDOR_ID_ADAPTEC2, 0x1441)
9800         },
9801         {
9802                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9803                                PCI_VENDOR_ID_ADAPTEC2, 0x1450)
9804         },
9805         {
9806                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9807                                PCI_VENDOR_ID_ADAPTEC2, 0x1452)
9808         },
9809         {
9810                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9811                                PCI_VENDOR_ID_ADAPTEC2, 0x1460)
9812         },
9813         {
9814                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9815                                PCI_VENDOR_ID_ADAPTEC2, 0x1461)
9816         },
9817         {
9818                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9819                                PCI_VENDOR_ID_ADAPTEC2, 0x1462)
9820         },
9821         {
9822                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9823                                PCI_VENDOR_ID_ADAPTEC2, 0x1463)
9824         },
9825         {
9826                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9827                                PCI_VENDOR_ID_ADAPTEC2, 0x1470)
9828         },
9829         {
9830                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9831                                PCI_VENDOR_ID_ADAPTEC2, 0x1471)
9832         },
9833         {
9834                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9835                                PCI_VENDOR_ID_ADAPTEC2, 0x1472)
9836         },
9837         {
9838                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9839                                PCI_VENDOR_ID_ADAPTEC2, 0x1473)
9840         },
9841         {
9842                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9843                                PCI_VENDOR_ID_ADAPTEC2, 0x1474)
9844         },
9845         {
9846                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9847                                PCI_VENDOR_ID_ADAPTEC2, 0x1475)
9848         },
9849         {
9850                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9851                                PCI_VENDOR_ID_ADAPTEC2, 0x1480)
9852         },
9853         {
9854                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9855                                PCI_VENDOR_ID_ADAPTEC2, 0x1490)
9856         },
9857         {
9858                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9859                                PCI_VENDOR_ID_ADAPTEC2, 0x1491)
9860         },
9861         {
9862                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9863                                PCI_VENDOR_ID_ADAPTEC2, 0x14a0)
9864         },
9865         {
9866                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9867                                PCI_VENDOR_ID_ADAPTEC2, 0x14a1)
9868         },
9869         {
9870                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9871                                PCI_VENDOR_ID_ADAPTEC2, 0x14a2)
9872         },
9873         {
9874                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9875                                PCI_VENDOR_ID_ADAPTEC2, 0x14a4)
9876         },
9877         {
9878                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9879                                PCI_VENDOR_ID_ADAPTEC2, 0x14a5)
9880         },
9881         {
9882                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9883                                PCI_VENDOR_ID_ADAPTEC2, 0x14a6)
9884         },
9885         {
9886                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9887                                PCI_VENDOR_ID_ADAPTEC2, 0x14b0)
9888         },
9889         {
9890                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9891                                PCI_VENDOR_ID_ADAPTEC2, 0x14b1)
9892         },
9893         {
9894                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9895                                PCI_VENDOR_ID_ADAPTEC2, 0x14c0)
9896         },
9897         {
9898                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9899                                PCI_VENDOR_ID_ADAPTEC2, 0x14c1)
9900         },
9901         {
9902                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9903                                PCI_VENDOR_ID_ADAPTEC2, 0x14c2)
9904         },
9905         {
9906                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9907                                PCI_VENDOR_ID_ADAPTEC2, 0x14c3)
9908         },
9909         {
9910                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9911                                PCI_VENDOR_ID_ADAPTEC2, 0x14c4)
9912         },
9913         {
9914                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9915                                PCI_VENDOR_ID_ADAPTEC2, 0x14d0)
9916         },
9917         {
9918                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9919                                PCI_VENDOR_ID_ADAPTEC2, 0x14e0)
9920         },
9921         {
9922                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9923                                PCI_VENDOR_ID_ADAPTEC2, 0x14f0)
9924         },
9925         {
9926                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9927                                PCI_VENDOR_ID_ADVANTECH, 0x8312)
9928         },
9929         {
9930                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9931                                PCI_VENDOR_ID_DELL, 0x1fe0)
9932         },
9933         {
9934                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9935                                PCI_VENDOR_ID_HP, 0x0600)
9936         },
9937         {
9938                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9939                                PCI_VENDOR_ID_HP, 0x0601)
9940         },
9941         {
9942                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9943                                PCI_VENDOR_ID_HP, 0x0602)
9944         },
9945         {
9946                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9947                                PCI_VENDOR_ID_HP, 0x0603)
9948         },
9949         {
9950                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9951                                PCI_VENDOR_ID_HP, 0x0609)
9952         },
9953         {
9954                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9955                                PCI_VENDOR_ID_HP, 0x0650)
9956         },
9957         {
9958                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9959                                PCI_VENDOR_ID_HP, 0x0651)
9960         },
9961         {
9962                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9963                                PCI_VENDOR_ID_HP, 0x0652)
9964         },
9965         {
9966                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9967                                PCI_VENDOR_ID_HP, 0x0653)
9968         },
9969         {
9970                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9971                                PCI_VENDOR_ID_HP, 0x0654)
9972         },
9973         {
9974                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9975                                PCI_VENDOR_ID_HP, 0x0655)
9976         },
9977         {
9978                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9979                                PCI_VENDOR_ID_HP, 0x0700)
9980         },
9981         {
9982                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9983                                PCI_VENDOR_ID_HP, 0x0701)
9984         },
9985         {
9986                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9987                                PCI_VENDOR_ID_HP, 0x1001)
9988         },
9989         {
9990                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9991                                PCI_VENDOR_ID_HP, 0x1002)
9992         },
9993         {
9994                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9995                                PCI_VENDOR_ID_HP, 0x1100)
9996         },
9997         {
9998                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9999                                PCI_VENDOR_ID_HP, 0x1101)
10000         },
10001         {
10002                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10003                                0x1590, 0x0294)
10004         },
10005         {
10006                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10007                                0x1590, 0x02db)
10008         },
10009         {
10010                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10011                                0x1590, 0x02dc)
10012         },
10013         {
10014                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10015                                0x1590, 0x032e)
10016         },
10017         {
10018                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10019                                0x1590, 0x036f)
10020         },
10021         {
10022                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10023                                0x1590, 0x0381)
10024         },
10025         {
10026                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10027                                0x1590, 0x0382)
10028         },
10029         {
10030                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10031                                0x1590, 0x0383)
10032         },
10033         {
10034                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10035                                0x1d8d, 0x0800)
10036         },
10037         {
10038                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10039                                0x1d8d, 0x0908)
10040         },
10041         {
10042                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10043                                0x1d8d, 0x0806)
10044         },
10045         {
10046                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10047                                0x1d8d, 0x0916)
10048         },
10049         {
10050                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10051                                PCI_VENDOR_ID_GIGABYTE, 0x1000)
10052         },
10053         {
10054                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10055                                0x1dfc, 0x3161)
10056         },
10057         {
10058                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10059                                0x1f0c, 0x3161)
10060         },
10061         {
10062                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10063                                0x1cf2, 0x0804)
10064         },
10065         {
10066                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10067                                0x1cf2, 0x0805)
10068         },
10069         {
10070                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10071                                0x1cf2, 0x0806)
10072         },
10073         {
10074                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10075                                0x1cf2, 0x5445)
10076         },
10077         {
10078                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10079                                0x1cf2, 0x5446)
10080         },
10081         {
10082                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10083                                0x1cf2, 0x5447)
10084         },
10085         {
10086                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10087                                0x1cf2, 0x5449)
10088         },
10089         {
10090                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10091                                0x1cf2, 0x544a)
10092         },
10093         {
10094                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10095                                0x1cf2, 0x544b)
10096         },
10097         {
10098                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10099                                0x1cf2, 0x544d)
10100         },
10101         {
10102                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10103                                0x1cf2, 0x544e)
10104         },
10105         {
10106                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10107                                0x1cf2, 0x544f)
10108         },
10109         {
10110                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10111                                0x1cf2, 0x54da)
10112         },
10113         {
10114                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10115                                0x1cf2, 0x54db)
10116         },
10117         {
10118                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10119                                0x1cf2, 0x54dc)
10120         },
10121         {
10122                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10123                                0x1cf2, 0x0b27)
10124         },
10125         {
10126                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10127                                0x1cf2, 0x0b29)
10128         },
10129         {
10130                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10131                                0x1cf2, 0x0b45)
10132         },
10133         {
10134                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10135                                0x1cc4, 0x0101)
10136         },
10137         {
10138                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10139                                0x1cc4, 0x0201)
10140         },
10141         {
10142                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10143                                PCI_VENDOR_ID_LENOVO, 0x0220)
10144         },
10145         {
10146                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10147                                PCI_VENDOR_ID_LENOVO, 0x0221)
10148         },
10149         {
10150                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10151                                PCI_VENDOR_ID_LENOVO, 0x0520)
10152         },
10153         {
10154                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10155                                PCI_VENDOR_ID_LENOVO, 0x0522)
10156         },
10157         {
10158                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10159                                PCI_VENDOR_ID_LENOVO, 0x0620)
10160         },
10161         {
10162                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10163                                PCI_VENDOR_ID_LENOVO, 0x0621)
10164         },
10165         {
10166                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10167                                PCI_VENDOR_ID_LENOVO, 0x0622)
10168         },
10169         {
10170                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10171                                PCI_VENDOR_ID_LENOVO, 0x0623)
10172         },
10173         {
10174                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10175                                 0x1014, 0x0718)
10176         },
10177         {
10178                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10179                                0x1137, 0x02f8)
10180         },
10181         {
10182                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10183                                0x1137, 0x02f9)
10184         },
10185         {
10186                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10187                                0x1137, 0x02fa)
10188         },
10189         {
10190                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10191                                 0x1e93, 0x1000)
10192         },
10193         {
10194                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10195                                 0x1e93, 0x1001)
10196         },
10197         {
10198                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10199                                 0x1e93, 0x1002)
10200         },
10201         {
10202                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10203                                 0x1e93, 0x1005)
10204         },
10205         {
10206                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10207                                 0x1f51, 0x1001)
10208         },
10209         {
10210                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10211                                 0x1f51, 0x1002)
10212         },
10213         {
10214                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10215                                 0x1f51, 0x1003)
10216         },
10217         {
10218                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10219                                 0x1f51, 0x1004)
10220         },
10221         {
10222                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10223                                 0x1f51, 0x1005)
10224         },
10225         {
10226                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10227                                 0x1f51, 0x1006)
10228         },
10229         {
10230                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10231                                 0x1f51, 0x1007)
10232         },
10233         {
10234                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10235                                 0x1f51, 0x1008)
10236         },
10237         {
10238                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10239                                 0x1f51, 0x1009)
10240         },
10241         {
10242                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10243                                 0x1f51, 0x100a)
10244         },
10245         {
10246                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10247                                0x1f51, 0x100e)
10248         },
10249         {
10250                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10251                                0x1f51, 0x100f)
10252         },
10253         {
10254                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10255                                0x1f51, 0x1010)
10256         },
10257         {
10258                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10259                                0x1f51, 0x1011)
10260         },
10261         {
10262                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10263                                0x1f51, 0x1043)
10264         },
10265         {
10266                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10267                                0x1f51, 0x1044)
10268         },
10269         {
10270                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10271                                0x1f51, 0x1045)
10272         },
10273         {
10274                 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10275                                PCI_ANY_ID, PCI_ANY_ID)
10276         },
10277         { 0 }
10278 };
10279
10280 MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
10281
10282 static struct pci_driver pqi_pci_driver = {
10283         .name = DRIVER_NAME_SHORT,
10284         .id_table = pqi_pci_id_table,
10285         .probe = pqi_pci_probe,
10286         .remove = pqi_pci_remove,
10287         .shutdown = pqi_shutdown,
10288 #if defined(CONFIG_PM)
10289         .driver = {
10290                 .pm = &pqi_pm_ops
10291         },
10292 #endif
10293 };
10294
10295 static int __init pqi_init(void)
10296 {
10297         int rc;
10298
10299         pr_info(DRIVER_NAME "\n");
10300         pqi_verify_structures();
10301         sis_verify_structures();
10302
10303         pqi_sas_transport_template = sas_attach_transport(&pqi_sas_transport_functions);
10304         if (!pqi_sas_transport_template)
10305                 return -ENODEV;
10306
10307         pqi_process_module_params();
10308
10309         rc = pci_register_driver(&pqi_pci_driver);
10310         if (rc)
10311                 sas_release_transport(pqi_sas_transport_template);
10312
10313         return rc;
10314 }
10315
10316 static void __exit pqi_cleanup(void)
10317 {
10318         pci_unregister_driver(&pqi_pci_driver);
10319         sas_release_transport(pqi_sas_transport_template);
10320 }
10321
10322 module_init(pqi_init);
10323 module_exit(pqi_cleanup);
10324
10325 static void pqi_verify_structures(void)
10326 {
10327         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10328                 sis_host_to_ctrl_doorbell) != 0x20);
10329         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10330                 sis_interrupt_mask) != 0x34);
10331         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10332                 sis_ctrl_to_host_doorbell) != 0x9c);
10333         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10334                 sis_ctrl_to_host_doorbell_clear) != 0xa0);
10335         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10336                 sis_driver_scratch) != 0xb0);
10337         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10338                 sis_product_identifier) != 0xb4);
10339         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10340                 sis_firmware_status) != 0xbc);
10341         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10342                 sis_ctrl_shutdown_reason_code) != 0xcc);
10343         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10344                 sis_mailbox) != 0x1000);
10345         BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10346                 pqi_registers) != 0x4000);
10347
10348         BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10349                 iu_type) != 0x0);
10350         BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10351                 iu_length) != 0x2);
10352         BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10353                 response_queue_id) != 0x4);
10354         BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10355                 driver_flags) != 0x6);
10356         BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
10357
10358         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10359                 status) != 0x0);
10360         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10361                 service_response) != 0x1);
10362         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10363                 data_present) != 0x2);
10364         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10365                 reserved) != 0x3);
10366         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10367                 residual_count) != 0x4);
10368         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10369                 data_length) != 0x8);
10370         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10371                 reserved1) != 0xa);
10372         BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10373                 data) != 0xc);
10374         BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
10375
10376         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10377                 data_in_result) != 0x0);
10378         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10379                 data_out_result) != 0x1);
10380         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10381                 reserved) != 0x2);
10382         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10383                 status) != 0x5);
10384         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10385                 status_qualifier) != 0x6);
10386         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10387                 sense_data_length) != 0x8);
10388         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10389                 response_data_length) != 0xa);
10390         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10391                 data_in_transferred) != 0xc);
10392         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10393                 data_out_transferred) != 0x10);
10394         BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10395                 data) != 0x14);
10396         BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
10397
10398         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10399                 signature) != 0x0);
10400         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10401                 function_and_status_code) != 0x8);
10402         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10403                 max_admin_iq_elements) != 0x10);
10404         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10405                 max_admin_oq_elements) != 0x11);
10406         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10407                 admin_iq_element_length) != 0x12);
10408         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10409                 admin_oq_element_length) != 0x13);
10410         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10411                 max_reset_timeout) != 0x14);
10412         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10413                 legacy_intx_status) != 0x18);
10414         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10415                 legacy_intx_mask_set) != 0x1c);
10416         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10417                 legacy_intx_mask_clear) != 0x20);
10418         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10419                 device_status) != 0x40);
10420         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10421                 admin_iq_pi_offset) != 0x48);
10422         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10423                 admin_oq_ci_offset) != 0x50);
10424         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10425                 admin_iq_element_array_addr) != 0x58);
10426         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10427                 admin_oq_element_array_addr) != 0x60);
10428         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10429                 admin_iq_ci_addr) != 0x68);
10430         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10431                 admin_oq_pi_addr) != 0x70);
10432         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10433                 admin_iq_num_elements) != 0x78);
10434         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10435                 admin_oq_num_elements) != 0x79);
10436         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10437                 admin_queue_int_msg_num) != 0x7a);
10438         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10439                 device_error) != 0x80);
10440         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10441                 error_details) != 0x88);
10442         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10443                 device_reset) != 0x90);
10444         BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10445                 power_action) != 0x94);
10446         BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
10447
10448         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10449                 header.iu_type) != 0);
10450         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10451                 header.iu_length) != 2);
10452         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10453                 header.driver_flags) != 6);
10454         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10455                 request_id) != 8);
10456         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10457                 function_code) != 10);
10458         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10459                 data.report_device_capability.buffer_length) != 44);
10460         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10461                 data.report_device_capability.sg_descriptor) != 48);
10462         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10463                 data.create_operational_iq.queue_id) != 12);
10464         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10465                 data.create_operational_iq.element_array_addr) != 16);
10466         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10467                 data.create_operational_iq.ci_addr) != 24);
10468         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10469                 data.create_operational_iq.num_elements) != 32);
10470         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10471                 data.create_operational_iq.element_length) != 34);
10472         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10473                 data.create_operational_iq.queue_protocol) != 36);
10474         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10475                 data.create_operational_oq.queue_id) != 12);
10476         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10477                 data.create_operational_oq.element_array_addr) != 16);
10478         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10479                 data.create_operational_oq.pi_addr) != 24);
10480         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10481                 data.create_operational_oq.num_elements) != 32);
10482         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10483                 data.create_operational_oq.element_length) != 34);
10484         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10485                 data.create_operational_oq.queue_protocol) != 36);
10486         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10487                 data.create_operational_oq.int_msg_num) != 40);
10488         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10489                 data.create_operational_oq.coalescing_count) != 42);
10490         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10491                 data.create_operational_oq.min_coalescing_time) != 44);
10492         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10493                 data.create_operational_oq.max_coalescing_time) != 48);
10494         BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10495                 data.delete_operational_queue.queue_id) != 12);
10496         BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
10497         BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10498                 data.create_operational_iq) != 64 - 11);
10499         BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10500                 data.create_operational_oq) != 64 - 11);
10501         BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10502                 data.delete_operational_queue) != 64 - 11);
10503
10504         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10505                 header.iu_type) != 0);
10506         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10507                 header.iu_length) != 2);
10508         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10509                 header.driver_flags) != 6);
10510         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10511                 request_id) != 8);
10512         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10513                 function_code) != 10);
10514         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10515                 status) != 11);
10516         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10517                 data.create_operational_iq.status_descriptor) != 12);
10518         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10519                 data.create_operational_iq.iq_pi_offset) != 16);
10520         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10521                 data.create_operational_oq.status_descriptor) != 12);
10522         BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10523                 data.create_operational_oq.oq_ci_offset) != 16);
10524         BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
10525
10526         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10527                 header.iu_type) != 0);
10528         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10529                 header.iu_length) != 2);
10530         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10531                 header.response_queue_id) != 4);
10532         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10533                 header.driver_flags) != 6);
10534         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10535                 request_id) != 8);
10536         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10537                 nexus_id) != 10);
10538         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10539                 buffer_length) != 12);
10540         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10541                 lun_number) != 16);
10542         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10543                 protocol_specific) != 24);
10544         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10545                 error_index) != 27);
10546         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10547                 cdb) != 32);
10548         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10549                 timeout) != 60);
10550         BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10551                 sg_descriptors) != 64);
10552         BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
10553                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
10554
10555         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10556                 header.iu_type) != 0);
10557         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10558                 header.iu_length) != 2);
10559         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10560                 header.response_queue_id) != 4);
10561         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10562                 header.driver_flags) != 6);
10563         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10564                 request_id) != 8);
10565         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10566                 nexus_id) != 12);
10567         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10568                 buffer_length) != 16);
10569         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10570                 data_encryption_key_index) != 22);
10571         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10572                 encrypt_tweak_lower) != 24);
10573         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10574                 encrypt_tweak_upper) != 28);
10575         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10576                 cdb) != 32);
10577         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10578                 error_index) != 48);
10579         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10580                 num_sg_descriptors) != 50);
10581         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10582                 cdb_length) != 51);
10583         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10584                 lun_number) != 52);
10585         BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10586                 sg_descriptors) != 64);
10587         BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
10588                 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
10589
10590         BUILD_BUG_ON(offsetof(struct pqi_io_response,
10591                 header.iu_type) != 0);
10592         BUILD_BUG_ON(offsetof(struct pqi_io_response,
10593                 header.iu_length) != 2);
10594         BUILD_BUG_ON(offsetof(struct pqi_io_response,
10595                 request_id) != 8);
10596         BUILD_BUG_ON(offsetof(struct pqi_io_response,
10597                 error_index) != 10);
10598
10599         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10600                 header.iu_type) != 0);
10601         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10602                 header.iu_length) != 2);
10603         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10604                 header.response_queue_id) != 4);
10605         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10606                 request_id) != 8);
10607         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10608                 data.report_event_configuration.buffer_length) != 12);
10609         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10610                 data.report_event_configuration.sg_descriptors) != 16);
10611         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10612                 data.set_event_configuration.global_event_oq_id) != 10);
10613         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10614                 data.set_event_configuration.buffer_length) != 12);
10615         BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10616                 data.set_event_configuration.sg_descriptors) != 16);
10617
10618         BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
10619                 max_inbound_iu_length) != 6);
10620         BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
10621                 max_outbound_iu_length) != 14);
10622         BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
10623
10624         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10625                 data_length) != 0);
10626         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10627                 iq_arbitration_priority_support_bitmask) != 8);
10628         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10629                 maximum_aw_a) != 9);
10630         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10631                 maximum_aw_b) != 10);
10632         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10633                 maximum_aw_c) != 11);
10634         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10635                 max_inbound_queues) != 16);
10636         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10637                 max_elements_per_iq) != 18);
10638         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10639                 max_iq_element_length) != 24);
10640         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10641                 min_iq_element_length) != 26);
10642         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10643                 max_outbound_queues) != 30);
10644         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10645                 max_elements_per_oq) != 32);
10646         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10647                 intr_coalescing_time_granularity) != 34);
10648         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10649                 max_oq_element_length) != 36);
10650         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10651                 min_oq_element_length) != 38);
10652         BUILD_BUG_ON(offsetof(struct pqi_device_capability,
10653                 iu_layer_descriptors) != 64);
10654         BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
10655
10656         BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
10657                 event_type) != 0);
10658         BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
10659                 oq_id) != 2);
10660         BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
10661
10662         BUILD_BUG_ON(offsetof(struct pqi_event_config,
10663                 num_event_descriptors) != 2);
10664         BUILD_BUG_ON(offsetof(struct pqi_event_config,
10665                 descriptors) != 4);
10666
10667         BUILD_BUG_ON(PQI_NUM_SUPPORTED_EVENTS !=
10668                 ARRAY_SIZE(pqi_supported_event_types));
10669
10670         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10671                 header.iu_type) != 0);
10672         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10673                 header.iu_length) != 2);
10674         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10675                 event_type) != 8);
10676         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10677                 event_id) != 10);
10678         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10679                 additional_event_id) != 12);
10680         BUILD_BUG_ON(offsetof(struct pqi_event_response,
10681                 data) != 16);
10682         BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
10683
10684         BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
10685                 header.iu_type) != 0);
10686         BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
10687                 header.iu_length) != 2);
10688         BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
10689                 event_type) != 8);
10690         BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
10691                 event_id) != 10);
10692         BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
10693                 additional_event_id) != 12);
10694         BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
10695
10696         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10697                 header.iu_type) != 0);
10698         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10699                 header.iu_length) != 2);
10700         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10701                 request_id) != 8);
10702         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10703                 nexus_id) != 10);
10704         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10705                 timeout) != 14);
10706         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10707                 lun_number) != 16);
10708         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10709                 protocol_specific) != 24);
10710         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10711                 outbound_queue_id_to_manage) != 26);
10712         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10713                 request_id_to_manage) != 28);
10714         BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
10715                 task_management_function) != 30);
10716         BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
10717
10718         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10719                 header.iu_type) != 0);
10720         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10721                 header.iu_length) != 2);
10722         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10723                 request_id) != 8);
10724         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10725                 nexus_id) != 10);
10726         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10727                 additional_response_info) != 12);
10728         BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
10729                 response_code) != 15);
10730         BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
10731
10732         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10733                 configured_logical_drive_count) != 0);
10734         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10735                 configuration_signature) != 1);
10736         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10737                 firmware_version_short) != 5);
10738         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10739                 extended_logical_unit_count) != 154);
10740         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10741                 firmware_build_number) != 190);
10742         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10743                 vendor_id) != 200);
10744         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10745                 product_id) != 208);
10746         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10747                 extra_controller_flags) != 286);
10748         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10749                 controller_mode) != 292);
10750         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10751                 spare_part_number) != 293);
10752         BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
10753                 firmware_version_long) != 325);
10754
10755         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10756                 phys_bay_in_box) != 115);
10757         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10758                 device_type) != 120);
10759         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10760                 redundant_path_present_map) != 1736);
10761         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10762                 active_path_number) != 1738);
10763         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10764                 alternate_paths_phys_connector) != 1739);
10765         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10766                 alternate_paths_phys_box_on_port) != 1755);
10767         BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
10768                 current_queue_depth_limit) != 1796);
10769         BUILD_BUG_ON(sizeof(struct bmic_identify_physical_device) != 2560);
10770
10771         BUILD_BUG_ON(sizeof(struct bmic_sense_feature_buffer_header) != 4);
10772         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
10773                 page_code) != 0);
10774         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
10775                 subpage_code) != 1);
10776         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
10777                 buffer_length) != 2);
10778
10779         BUILD_BUG_ON(sizeof(struct bmic_sense_feature_page_header) != 4);
10780         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
10781                 page_code) != 0);
10782         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
10783                 subpage_code) != 1);
10784         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
10785                 page_length) != 2);
10786
10787         BUILD_BUG_ON(sizeof(struct bmic_sense_feature_io_page_aio_subpage)
10788                 != 18);
10789         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10790                 header) != 0);
10791         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10792                 firmware_read_support) != 4);
10793         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10794                 driver_read_support) != 5);
10795         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10796                 firmware_write_support) != 6);
10797         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10798                 driver_write_support) != 7);
10799         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10800                 max_transfer_encrypted_sas_sata) != 8);
10801         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10802                 max_transfer_encrypted_nvme) != 10);
10803         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10804                 max_write_raid_5_6) != 12);
10805         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10806                 max_write_raid_1_10_2drive) != 14);
10807         BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
10808                 max_write_raid_1_10_3drive) != 16);
10809
10810         BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
10811         BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
10812         BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
10813                 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
10814         BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
10815                 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
10816         BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
10817         BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
10818                 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
10819         BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
10820         BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
10821                 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
10822
10823         BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
10824         BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >=
10825                 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP);
10826 }