drivers/scsi/hisi_sas/hisi_sas_main.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (c) 2015 Linaro Ltd.
   4  * Copyright (c) 2015 Hisilicon Limited.
   5  */
   6
   7 #include "hisi_sas.h"
   8 #define DRV_NAME "hisi_sas"
   9
  10 #define DEV_IS_GONE(dev) \
  11         ((!dev) || (dev->dev_type == SAS_PHY_UNUSED))
  12
  13 static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device,
  14                                 u8 *lun, struct hisi_sas_tmf_task *tmf);
  15 static int
  16 hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
  17                              struct domain_device *device,
  18                              int abort_flag, int tag);
  19 static int hisi_sas_softreset_ata_disk(struct domain_device *device);
  20 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
  21                                 void *funcdata);
  22 static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
  23                                   struct domain_device *device);
  24 static void hisi_sas_dev_gone(struct domain_device *device);
  25
  26 u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction)
  27 {
  28         switch (fis->command) {
  29         case ATA_CMD_FPDMA_WRITE:
  30         case ATA_CMD_FPDMA_READ:
  31         case ATA_CMD_FPDMA_RECV:
  32         case ATA_CMD_FPDMA_SEND:
  33         case ATA_CMD_NCQ_NON_DATA:
  34                 return HISI_SAS_SATA_PROTOCOL_FPDMA;
  35
  36         case ATA_CMD_DOWNLOAD_MICRO:
  37         case ATA_CMD_ID_ATA:
  38         case ATA_CMD_PMP_READ:
  39         case ATA_CMD_READ_LOG_EXT:
  40         case ATA_CMD_PIO_READ:
  41         case ATA_CMD_PIO_READ_EXT:
  42         case ATA_CMD_PMP_WRITE:
  43         case ATA_CMD_WRITE_LOG_EXT:
  44         case ATA_CMD_PIO_WRITE:
  45         case ATA_CMD_PIO_WRITE_EXT:
  46                 return HISI_SAS_SATA_PROTOCOL_PIO;
  47
  48         case ATA_CMD_DSM:
  49         case ATA_CMD_DOWNLOAD_MICRO_DMA:
  50         case ATA_CMD_PMP_READ_DMA:
  51         case ATA_CMD_PMP_WRITE_DMA:
  52         case ATA_CMD_READ:
  53         case ATA_CMD_READ_EXT:
  54         case ATA_CMD_READ_LOG_DMA_EXT:
  55         case ATA_CMD_READ_STREAM_DMA_EXT:
  56         case ATA_CMD_TRUSTED_RCV_DMA:
  57         case ATA_CMD_TRUSTED_SND_DMA:
  58         case ATA_CMD_WRITE:
  59         case ATA_CMD_WRITE_EXT:
  60         case ATA_CMD_WRITE_FUA_EXT:
  61         case ATA_CMD_WRITE_QUEUED:
  62         case ATA_CMD_WRITE_LOG_DMA_EXT:
  63         case ATA_CMD_WRITE_STREAM_DMA_EXT:
  64         case ATA_CMD_ZAC_MGMT_IN:
  65                 return HISI_SAS_SATA_PROTOCOL_DMA;
  66
  67         case ATA_CMD_CHK_POWER:
  68         case ATA_CMD_DEV_RESET:
  69         case ATA_CMD_EDD:
  70         case ATA_CMD_FLUSH:
  71         case ATA_CMD_FLUSH_EXT:
  72         case ATA_CMD_VERIFY:
  73         case ATA_CMD_VERIFY_EXT:
  74         case ATA_CMD_SET_FEATURES:
  75         case ATA_CMD_STANDBY:
  76         case ATA_CMD_STANDBYNOW1:
  77         case ATA_CMD_ZAC_MGMT_OUT:
  78                 return HISI_SAS_SATA_PROTOCOL_NONDATA;
  79
  80         case ATA_CMD_SET_MAX:
  81                 switch (fis->features) {
  82                 case ATA_SET_MAX_PASSWD:
  83                 case ATA_SET_MAX_LOCK:
  84                         return HISI_SAS_SATA_PROTOCOL_PIO;
  85
  86                 case ATA_SET_MAX_PASSWD_DMA:
  87                 case ATA_SET_MAX_UNLOCK_DMA:
  88                         return HISI_SAS_SATA_PROTOCOL_DMA;
  89
  90                 default:
  91                         return HISI_SAS_SATA_PROTOCOL_NONDATA;
  92                 }
  93
  94         default:
  95         {
  96                 if (direction == DMA_NONE)
  97                         return HISI_SAS_SATA_PROTOCOL_NONDATA;
  98                 return HISI_SAS_SATA_PROTOCOL_PIO;
  99         }
 100         }
 101 }
 102 EXPORT_SYMBOL_GPL(hisi_sas_get_ata_protocol);
 103
 104 void hisi_sas_sata_done(struct sas_task *task,
 105                             struct hisi_sas_slot *slot)
 106 {
 107         struct task_status_struct *ts = &task->task_status;
 108         struct ata_task_resp *resp = (struct ata_task_resp *)ts->buf;
 109         struct hisi_sas_status_buffer *status_buf =
 110                         hisi_sas_status_buf_addr_mem(slot);
 111         u8 *iu = &status_buf->iu[0];
 112         struct dev_to_host_fis *d2h =  (struct dev_to_host_fis *)iu;
 113
 114         resp->frame_len = sizeof(struct dev_to_host_fis);
 115         memcpy(&resp->ending_fis[0], d2h, sizeof(struct dev_to_host_fis));
 116
 117         ts->buf_valid_size = sizeof(*resp);
 118 }
 119 EXPORT_SYMBOL_GPL(hisi_sas_sata_done);
 120
 121 /*
 122  * This function assumes linkrate mask fits in 8 bits, which it
 123  * does for all HW versions supported.
 124  */
 125 u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max)
 126 {
 127         u8 rate = 0;
 128         int i;
 129
 130         max -= SAS_LINK_RATE_1_5_GBPS;
 131         for (i = 0; i <= max; i++)
 132                 rate |= 1 << (i * 2);
 133         return rate;
 134 }
 135 EXPORT_SYMBOL_GPL(hisi_sas_get_prog_phy_linkrate_mask);
 136
 137 static struct hisi_hba *dev_to_hisi_hba(struct domain_device *device)
 138 {
 139         return device->port->ha->lldd_ha;
 140 }
 141
 142 struct hisi_sas_port *to_hisi_sas_port(struct asd_sas_port *sas_port)
 143 {
 144         return container_of(sas_port, struct hisi_sas_port, sas_port);
 145 }
 146 EXPORT_SYMBOL_GPL(to_hisi_sas_port);
 147
 148 void hisi_sas_stop_phys(struct hisi_hba *hisi_hba)
 149 {
 150         int phy_no;
 151
 152         for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++)
 153                 hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 154 }
 155 EXPORT_SYMBOL_GPL(hisi_sas_stop_phys);
 156
 157 static void hisi_sas_slot_index_clear(struct hisi_hba *hisi_hba, int slot_idx)
 158 {
 159         void *bitmap = hisi_hba->slot_index_tags;
 160
 161         clear_bit(slot_idx, bitmap);
 162 }
 163
 164 static void hisi_sas_slot_index_free(struct hisi_hba *hisi_hba, int slot_idx)
 165 {
 166         unsigned long flags;
 167
 168         if (hisi_hba->hw->slot_index_alloc ||
 169             slot_idx >= HISI_SAS_UNRESERVED_IPTT) {
 170                 spin_lock_irqsave(&hisi_hba->lock, flags);
 171                 hisi_sas_slot_index_clear(hisi_hba, slot_idx);
 172                 spin_unlock_irqrestore(&hisi_hba->lock, flags);
 173         }
 174 }
 175
 176 static void hisi_sas_slot_index_set(struct hisi_hba *hisi_hba, int slot_idx)
 177 {
 178         void *bitmap = hisi_hba->slot_index_tags;
 179
 180         set_bit(slot_idx, bitmap);
 181 }
 182
 183 static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba,
 184                                      struct scsi_cmnd *scsi_cmnd)
 185 {
 186         int index;
 187         void *bitmap = hisi_hba->slot_index_tags;
 188         unsigned long flags;
 189
 190         if (scsi_cmnd)
 191                 return scsi_cmnd->request->tag;
 192
 193         spin_lock_irqsave(&hisi_hba->lock, flags);
 194         index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count,
 195                                    hisi_hba->last_slot_index + 1);
 196         if (index >= hisi_hba->slot_index_count) {
 197                 index = find_next_zero_bit(bitmap,
 198                                 hisi_hba->slot_index_count,
 199                                 HISI_SAS_UNRESERVED_IPTT);
 200                 if (index >= hisi_hba->slot_index_count) {
 201                         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 202                         return -SAS_QUEUE_FULL;
 203                 }
 204         }
 205         hisi_sas_slot_index_set(hisi_hba, index);
 206         hisi_hba->last_slot_index = index;
 207         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 208
 209         return index;
 210 }
 211
 212 static void hisi_sas_slot_index_init(struct hisi_hba *hisi_hba)
 213 {
 214         int i;
 215
 216         for (i = 0; i < hisi_hba->slot_index_count; ++i)
 217                 hisi_sas_slot_index_clear(hisi_hba, i);
 218 }
 219
 220 void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
 221                              struct hisi_sas_slot *slot)
 222 {
 223         unsigned long flags;
 224         int device_id = slot->device_id;
 225         struct hisi_sas_device *sas_dev = &hisi_hba->devices[device_id];
 226
 227         if (task) {
 228                 struct device *dev = hisi_hba->dev;
 229
 230                 if (!task->lldd_task)
 231                         return;
 232
 233                 task->lldd_task = NULL;
 234
 235                 if (!sas_protocol_ata(task->task_proto)) {
 236                         struct sas_ssp_task *ssp_task = &task->ssp_task;
 237                         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 238
 239                         if (slot->n_elem)
 240                                 dma_unmap_sg(dev, task->scatter,
 241                                              task->num_scatter,
 242                                              task->data_dir);
 243                         if (slot->n_elem_dif)
 244                                 dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd),
 245                                              scsi_prot_sg_count(scsi_cmnd),
 246                                              task->data_dir);
 247                 }
 248         }
 249
 250         spin_lock_irqsave(&sas_dev->lock, flags);
 251         list_del_init(&slot->entry);
 252         spin_unlock_irqrestore(&sas_dev->lock, flags);
 253
 254         memset(slot, 0, offsetof(struct hisi_sas_slot, buf));
 255
 256         hisi_sas_slot_index_free(hisi_hba, slot->idx);
 257 }
 258 EXPORT_SYMBOL_GPL(hisi_sas_slot_task_free);
 259
 260 static void hisi_sas_task_prep_smp(struct hisi_hba *hisi_hba,
 261                                   struct hisi_sas_slot *slot)
 262 {
 263         hisi_hba->hw->prep_smp(hisi_hba, slot);
 264 }
 265
 266 static void hisi_sas_task_prep_ssp(struct hisi_hba *hisi_hba,
 267                                   struct hisi_sas_slot *slot)
 268 {
 269         hisi_hba->hw->prep_ssp(hisi_hba, slot);
 270 }
 271
 272 static void hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
 273                                   struct hisi_sas_slot *slot)
 274 {
 275         hisi_hba->hw->prep_stp(hisi_hba, slot);
 276 }
 277
 278 static void hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
 279                 struct hisi_sas_slot *slot,
 280                 int device_id, int abort_flag, int tag_to_abort)
 281 {
 282         hisi_hba->hw->prep_abort(hisi_hba, slot,
 283                         device_id, abort_flag, tag_to_abort);
 284 }
 285
 286 static void hisi_sas_dma_unmap(struct hisi_hba *hisi_hba,
 287                                struct sas_task *task, int n_elem,
 288                                int n_elem_req)
 289 {
 290         struct device *dev = hisi_hba->dev;
 291
 292         if (!sas_protocol_ata(task->task_proto)) {
 293                 if (task->num_scatter) {
 294                         if (n_elem)
 295                                 dma_unmap_sg(dev, task->scatter,
 296                                              task->num_scatter,
 297                                              task->data_dir);
 298                 } else if (task->task_proto & SAS_PROTOCOL_SMP) {
 299                         if (n_elem_req)
 300                                 dma_unmap_sg(dev, &task->smp_task.smp_req,
 301                                              1, DMA_TO_DEVICE);
 302                 }
 303         }
 304 }
 305
 306 static int hisi_sas_dma_map(struct hisi_hba *hisi_hba,
 307                             struct sas_task *task, int *n_elem,
 308                             int *n_elem_req)
 309 {
 310         struct device *dev = hisi_hba->dev;
 311         int rc;
 312
 313         if (sas_protocol_ata(task->task_proto)) {
 314                 *n_elem = task->num_scatter;
 315         } else {
 316                 unsigned int req_len;
 317
 318                 if (task->num_scatter) {
 319                         *n_elem = dma_map_sg(dev, task->scatter,
 320                                              task->num_scatter, task->data_dir);
 321                         if (!*n_elem) {
 322                                 rc = -ENOMEM;
 323                                 goto prep_out;
 324                         }
 325                 } else if (task->task_proto & SAS_PROTOCOL_SMP) {
 326                         *n_elem_req = dma_map_sg(dev, &task->smp_task.smp_req,
 327                                                  1, DMA_TO_DEVICE);
 328                         if (!*n_elem_req) {
 329                                 rc = -ENOMEM;
 330                                 goto prep_out;
 331                         }
 332                         req_len = sg_dma_len(&task->smp_task.smp_req);
 333                         if (req_len & 0x3) {
 334                                 rc = -EINVAL;
 335                                 goto err_out_dma_unmap;
 336                         }
 337                 }
 338         }
 339
 340         if (*n_elem > HISI_SAS_SGE_PAGE_CNT) {
 341                 dev_err(dev, "task prep: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
 342                         *n_elem);
 343                 rc = -EINVAL;
 344                 goto err_out_dma_unmap;
 345         }
 346         return 0;
 347
 348 err_out_dma_unmap:
 349         /* It would be better to call dma_unmap_sg() here, but it's messy */
 350         hisi_sas_dma_unmap(hisi_hba, task, *n_elem,
 351                            *n_elem_req);
 352 prep_out:
 353         return rc;
 354 }
 355
 356 static void hisi_sas_dif_dma_unmap(struct hisi_hba *hisi_hba,
 357                                    struct sas_task *task, int n_elem_dif)
 358 {
 359         struct device *dev = hisi_hba->dev;
 360
 361         if (n_elem_dif) {
 362                 struct sas_ssp_task *ssp_task = &task->ssp_task;
 363                 struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 364
 365                 dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd),
 366                              scsi_prot_sg_count(scsi_cmnd),
 367                              task->data_dir);
 368         }
 369 }
 370
 371 static int hisi_sas_dif_dma_map(struct hisi_hba *hisi_hba,
 372                                 int *n_elem_dif, struct sas_task *task)
 373 {
 374         struct device *dev = hisi_hba->dev;
 375         struct sas_ssp_task *ssp_task;
 376         struct scsi_cmnd *scsi_cmnd;
 377         int rc;
 378
 379         if (task->num_scatter) {
 380                 ssp_task = &task->ssp_task;
 381                 scsi_cmnd = ssp_task->cmd;
 382
 383                 if (scsi_prot_sg_count(scsi_cmnd)) {
 384                         *n_elem_dif = dma_map_sg(dev,
 385                                                  scsi_prot_sglist(scsi_cmnd),
 386                                                  scsi_prot_sg_count(scsi_cmnd),
 387                                                  task->data_dir);
 388
 389                         if (!*n_elem_dif)
 390                                 return -ENOMEM;
 391
 392                         if (*n_elem_dif > HISI_SAS_SGE_DIF_PAGE_CNT) {
 393                                 dev_err(dev, "task prep: n_elem_dif(%d) too large\n",
 394                                         *n_elem_dif);
 395                                 rc = -EINVAL;
 396                                 goto err_out_dif_dma_unmap;
 397                         }
 398                 }
 399         }
 400
 401         return 0;
 402
 403 err_out_dif_dma_unmap:
 404         dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd),
 405                      scsi_prot_sg_count(scsi_cmnd), task->data_dir);
 406         return rc;
 407 }
 408
 409 static int hisi_sas_task_prep(struct sas_task *task,
 410                               struct hisi_sas_dq **dq_pointer,
 411                               bool is_tmf, struct hisi_sas_tmf_task *tmf,
 412                               int *pass)
 413 {
 414         struct domain_device *device = task->dev;
 415         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 416         struct hisi_sas_device *sas_dev = device->lldd_dev;
 417         struct hisi_sas_port *port;
 418         struct hisi_sas_slot *slot;
 419         struct hisi_sas_cmd_hdr *cmd_hdr_base;
 420         struct asd_sas_port *sas_port = device->port;
 421         struct device *dev = hisi_hba->dev;
 422         int dlvry_queue_slot, dlvry_queue, rc, slot_idx;
 423         int n_elem = 0, n_elem_dif = 0, n_elem_req = 0;
 424         struct hisi_sas_dq *dq;
 425         unsigned long flags;
 426         int wr_q_index;
 427
 428         if (DEV_IS_GONE(sas_dev)) {
 429                 if (sas_dev)
 430                         dev_info(dev, "task prep: device %d not ready\n",
 431                                  sas_dev->device_id);
 432                 else
 433                         dev_info(dev, "task prep: device %016llx not ready\n",
 434                                  SAS_ADDR(device->sas_addr));
 435
 436                 return -ECOMM;
 437         }
 438
 439         if (hisi_hba->reply_map) {
 440                 int cpu = raw_smp_processor_id();
 441                 unsigned int dq_index = hisi_hba->reply_map[cpu];
 442
 443                 *dq_pointer = dq = &hisi_hba->dq[dq_index];
 444         } else {
 445                 *dq_pointer = dq = sas_dev->dq;
 446         }
 447
 448         port = to_hisi_sas_port(sas_port);
 449         if (port && !port->port_attached) {
 450                 dev_info(dev, "task prep: %s port%d not attach device\n",
 451                          (dev_is_sata(device)) ?
 452                          "SATA/STP" : "SAS",
 453                          device->port->id);
 454
 455                 return -ECOMM;
 456         }
 457
 458         rc = hisi_sas_dma_map(hisi_hba, task, &n_elem,
 459                               &n_elem_req);
 460         if (rc < 0)
 461                 goto prep_out;
 462
 463         if (!sas_protocol_ata(task->task_proto)) {
 464                 rc = hisi_sas_dif_dma_map(hisi_hba, &n_elem_dif, task);
 465                 if (rc < 0)
 466                         goto err_out_dma_unmap;
 467         }
 468
 469         if (hisi_hba->hw->slot_index_alloc)
 470                 rc = hisi_hba->hw->slot_index_alloc(hisi_hba, device);
 471         else {
 472                 struct scsi_cmnd *scsi_cmnd = NULL;
 473
 474                 if (task->uldd_task) {
 475                         struct ata_queued_cmd *qc;
 476
 477                         if (dev_is_sata(device)) {
 478                                 qc = task->uldd_task;
 479                                 scsi_cmnd = qc->scsicmd;
 480                         } else {
 481                                 scsi_cmnd = task->uldd_task;
 482                         }
 483                 }
 484                 rc  = hisi_sas_slot_index_alloc(hisi_hba, scsi_cmnd);
 485         }
 486         if (rc < 0)
 487                 goto err_out_dif_dma_unmap;
 488
 489         slot_idx = rc;
 490         slot = &hisi_hba->slot_info[slot_idx];
 491
 492         spin_lock_irqsave(&dq->lock, flags);
 493         wr_q_index = dq->wr_point;
 494         dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
 495         list_add_tail(&slot->delivery, &dq->list);
 496         spin_unlock_irqrestore(&dq->lock, flags);
 497         spin_lock_irqsave(&sas_dev->lock, flags);
 498         list_add_tail(&slot->entry, &sas_dev->list);
 499         spin_unlock_irqrestore(&sas_dev->lock, flags);
 500
 501         dlvry_queue = dq->id;
 502         dlvry_queue_slot = wr_q_index;
 503
 504         slot->device_id = sas_dev->device_id;
 505         slot->n_elem = n_elem;
 506         slot->n_elem_dif = n_elem_dif;
 507         slot->dlvry_queue = dlvry_queue;
 508         slot->dlvry_queue_slot = dlvry_queue_slot;
 509         cmd_hdr_base = hisi_hba->cmd_hdr[dlvry_queue];
 510         slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
 511         slot->task = task;
 512         slot->port = port;
 513         slot->tmf = tmf;
 514         slot->is_internal = is_tmf;
 515         task->lldd_task = slot;
 516
 517         memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
 518         memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
 519         memset(hisi_sas_status_buf_addr_mem(slot), 0,
 520                sizeof(struct hisi_sas_err_record));
 521
 522         switch (task->task_proto) {
 523         case SAS_PROTOCOL_SMP:
 524                 hisi_sas_task_prep_smp(hisi_hba, slot);
 525                 break;
 526         case SAS_PROTOCOL_SSP:
 527                 hisi_sas_task_prep_ssp(hisi_hba, slot);
 528                 break;
 529         case SAS_PROTOCOL_SATA:
 530         case SAS_PROTOCOL_STP:
 531         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
 532                 hisi_sas_task_prep_ata(hisi_hba, slot);
 533                 break;
 534         default:
 535                 dev_err(dev, "task prep: unknown/unsupported proto (0x%x)\n",
 536                         task->task_proto);
 537                 break;
 538         }
 539
 540         spin_lock_irqsave(&task->task_state_lock, flags);
 541         task->task_state_flags |= SAS_TASK_AT_INITIATOR;
 542         spin_unlock_irqrestore(&task->task_state_lock, flags);
 543
 544         ++(*pass);
 545         WRITE_ONCE(slot->ready, 1);
 546
 547         return 0;
 548
 549 err_out_dif_dma_unmap:
 550         if (!sas_protocol_ata(task->task_proto))
 551                 hisi_sas_dif_dma_unmap(hisi_hba, task, n_elem_dif);
 552 err_out_dma_unmap:
 553         hisi_sas_dma_unmap(hisi_hba, task, n_elem,
 554                            n_elem_req);
 555 prep_out:
 556         dev_err(dev, "task prep: failed[%d]!\n", rc);
 557         return rc;
 558 }
 559
 560 static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags,
 561                               bool is_tmf, struct hisi_sas_tmf_task *tmf)
 562 {
 563         u32 rc;
 564         u32 pass = 0;
 565         unsigned long flags;
 566         struct hisi_hba *hisi_hba;
 567         struct device *dev;
 568         struct domain_device *device = task->dev;
 569         struct asd_sas_port *sas_port = device->port;
 570         struct hisi_sas_dq *dq = NULL;
 571
 572         if (!sas_port) {
 573                 struct task_status_struct *ts = &task->task_status;
 574
 575                 ts->resp = SAS_TASK_UNDELIVERED;
 576                 ts->stat = SAS_PHY_DOWN;
 577                 /*
 578                  * libsas will use dev->port, should
 579                  * not call task_done for sata
 580                  */
 581                 if (device->dev_type != SAS_SATA_DEV)
 582                         task->task_done(task);
 583                 return -ECOMM;
 584         }
 585
 586         hisi_hba = dev_to_hisi_hba(device);
 587         dev = hisi_hba->dev;
 588
 589         if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags))) {
 590                 /*
 591                  * For IOs from upper layer, it may already disable preempt
 592                  * in the IO path, if disable preempt again in down(),
 593                  * function schedule() will report schedule_bug(), so check
 594                  * preemptible() before goto down().
 595                  */
 596                 if (!preemptible())
 597                         return -EINVAL;
 598
 599                 down(&hisi_hba->sem);
 600                 up(&hisi_hba->sem);
 601         }
 602
 603         /* protect task_prep and start_delivery sequence */
 604         rc = hisi_sas_task_prep(task, &dq, is_tmf, tmf, &pass);
 605         if (rc)
 606                 dev_err(dev, "task exec: failed[%d]!\n", rc);
 607
 608         if (likely(pass)) {
 609                 spin_lock_irqsave(&dq->lock, flags);
 610                 hisi_hba->hw->start_delivery(dq);
 611                 spin_unlock_irqrestore(&dq->lock, flags);
 612         }
 613
 614         return rc;
 615 }
 616
 617 static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no)
 618 {
 619         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 620         struct asd_sas_phy *sas_phy = &phy->sas_phy;
 621         struct sas_ha_struct *sas_ha;
 622
 623         if (!phy->phy_attached)
 624                 return;
 625
 626         sas_ha = &hisi_hba->sha;
 627         sas_ha->notify_phy_event(sas_phy, PHYE_OOB_DONE);
 628
 629         if (sas_phy->phy) {
 630                 struct sas_phy *sphy = sas_phy->phy;
 631
 632                 sphy->negotiated_linkrate = sas_phy->linkrate;
 633                 sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS;
 634                 sphy->maximum_linkrate_hw =
 635                         hisi_hba->hw->phy_get_max_linkrate();
 636                 if (sphy->minimum_linkrate == SAS_LINK_RATE_UNKNOWN)
 637                         sphy->minimum_linkrate = phy->minimum_linkrate;
 638
 639                 if (sphy->maximum_linkrate == SAS_LINK_RATE_UNKNOWN)
 640                         sphy->maximum_linkrate = phy->maximum_linkrate;
 641         }
 642
 643         if (phy->phy_type & PORT_TYPE_SAS) {
 644                 struct sas_identify_frame *id;
 645
 646                 id = (struct sas_identify_frame *)phy->frame_rcvd;
 647                 id->dev_type = phy->identify.device_type;
 648                 id->initiator_bits = SAS_PROTOCOL_ALL;
 649                 id->target_bits = phy->identify.target_port_protocols;
 650         } else if (phy->phy_type & PORT_TYPE_SATA) {
 651                 /* Nothing */
 652         }
 653
 654         sas_phy->frame_rcvd_size = phy->frame_rcvd_size;
 655         sas_ha->notify_port_event(sas_phy, PORTE_BYTES_DMAED);
 656 }
 657
 658 static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
 659 {
 660         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 661         struct hisi_sas_device *sas_dev = NULL;
 662         unsigned long flags;
 663         int last = hisi_hba->last_dev_id;
 664         int first = (hisi_hba->last_dev_id + 1) % HISI_SAS_MAX_DEVICES;
 665         int i;
 666
 667         spin_lock_irqsave(&hisi_hba->lock, flags);
 668         for (i = first; i != last; i %= HISI_SAS_MAX_DEVICES) {
 669                 if (hisi_hba->devices[i].dev_type == SAS_PHY_UNUSED) {
 670                         int queue = i % hisi_hba->queue_count;
 671                         struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
 672
 673                         hisi_hba->devices[i].device_id = i;
 674                         sas_dev = &hisi_hba->devices[i];
 675                         sas_dev->dev_status = HISI_SAS_DEV_INIT;
 676                         sas_dev->dev_type = device->dev_type;
 677                         sas_dev->hisi_hba = hisi_hba;
 678                         sas_dev->sas_device = device;
 679                         sas_dev->dq = dq;
 680                         spin_lock_init(&sas_dev->lock);
 681                         INIT_LIST_HEAD(&hisi_hba->devices[i].list);
 682                         break;
 683                 }
 684                 i++;
 685         }
 686         hisi_hba->last_dev_id = i;
 687         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 688
 689         return sas_dev;
 690 }
 691
 692 #define HISI_SAS_DISK_RECOVER_CNT 3
 693 static int hisi_sas_init_device(struct domain_device *device)
 694 {
 695         int rc = TMF_RESP_FUNC_COMPLETE;
 696         struct scsi_lun lun;
 697         struct hisi_sas_tmf_task tmf_task;
 698         int retry = HISI_SAS_DISK_RECOVER_CNT;
 699         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 700         struct device *dev = hisi_hba->dev;
 701         struct sas_phy *local_phy;
 702
 703         switch (device->dev_type) {
 704         case SAS_END_DEVICE:
 705                 int_to_scsilun(0, &lun);
 706
 707                 tmf_task.tmf = TMF_CLEAR_TASK_SET;
 708                 while (retry-- > 0) {
 709                         rc = hisi_sas_debug_issue_ssp_tmf(device, lun.scsi_lun,
 710                                                           &tmf_task);
 711                         if (rc == TMF_RESP_FUNC_COMPLETE) {
 712                                 hisi_sas_release_task(hisi_hba, device);
 713                                 break;
 714                         }
 715                 }
 716                 break;
 717         case SAS_SATA_DEV:
 718         case SAS_SATA_PM:
 719         case SAS_SATA_PM_PORT:
 720         case SAS_SATA_PENDING:
 721                 /*
 722                  * send HARD RESET to clear previous affiliation of
 723                  * STP target port
 724                  */
 725                 local_phy = sas_get_local_phy(device);
 726                 if (!scsi_is_sas_phy_local(local_phy) &&
 727                     !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) {
 728                         unsigned long deadline = ata_deadline(jiffies, 20000);
 729                         struct sata_device *sata_dev = &device->sata_dev;
 730                         struct ata_host *ata_host = sata_dev->ata_host;
 731                         struct ata_port_operations *ops = ata_host->ops;
 732                         struct ata_port *ap = sata_dev->ap;
 733                         struct ata_link *link;
 734                         unsigned int classes;
 735
 736                         ata_for_each_link(link, ap, EDGE)
 737                                 rc = ops->hardreset(link, &classes,
 738                                                     deadline);
 739                 }
 740                 sas_put_local_phy(local_phy);
 741                 if (rc) {
 742                         dev_warn(dev, "SATA disk hardreset fail: %d\n", rc);
 743                         return rc;
 744                 }
 745
 746                 while (retry-- > 0) {
 747                         rc = hisi_sas_softreset_ata_disk(device);
 748                         if (!rc)
 749                                 break;
 750                 }
 751                 break;
 752         default:
 753                 break;
 754         }
 755
 756         return rc;
 757 }
 758
 759 static int hisi_sas_dev_found(struct domain_device *device)
 760 {
 761         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 762         struct domain_device *parent_dev = device->parent;
 763         struct hisi_sas_device *sas_dev;
 764         struct device *dev = hisi_hba->dev;
 765         int rc;
 766
 767         if (hisi_hba->hw->alloc_dev)
 768                 sas_dev = hisi_hba->hw->alloc_dev(device);
 769         else
 770                 sas_dev = hisi_sas_alloc_dev(device);
 771         if (!sas_dev) {
 772                 dev_err(dev, "fail alloc dev: max support %d devices\n",
 773                         HISI_SAS_MAX_DEVICES);
 774                 return -EINVAL;
 775         }
 776
 777         device->lldd_dev = sas_dev;
 778         hisi_hba->hw->setup_itct(hisi_hba, sas_dev);
 779
 780         if (parent_dev && dev_is_expander(parent_dev->dev_type)) {
 781                 int phy_no;
 782                 u8 phy_num = parent_dev->ex_dev.num_phys;
 783                 struct ex_phy *phy;
 784
 785                 for (phy_no = 0; phy_no < phy_num; phy_no++) {
 786                         phy = &parent_dev->ex_dev.ex_phy[phy_no];
 787                         if (SAS_ADDR(phy->attached_sas_addr) ==
 788                                 SAS_ADDR(device->sas_addr))
 789                                 break;
 790                 }
 791
 792                 if (phy_no == phy_num) {
 793                         dev_info(dev, "dev found: no attached "
 794                                  "dev:%016llx at ex:%016llx\n",
 795                                  SAS_ADDR(device->sas_addr),
 796                                  SAS_ADDR(parent_dev->sas_addr));
 797                         rc = -EINVAL;
 798                         goto err_out;
 799                 }
 800         }
 801
 802         dev_info(dev, "dev[%d:%x] found\n",
 803                 sas_dev->device_id, sas_dev->dev_type);
 804
 805         rc = hisi_sas_init_device(device);
 806         if (rc)
 807                 goto err_out;
 808         sas_dev->dev_status = HISI_SAS_DEV_NORMAL;
 809         return 0;
 810
 811 err_out:
 812         hisi_sas_dev_gone(device);
 813         return rc;
 814 }
 815
 816 int hisi_sas_slave_configure(struct scsi_device *sdev)
 817 {
 818         struct domain_device *dev = sdev_to_domain_dev(sdev);
 819         int ret = sas_slave_configure(sdev);
 820
 821         if (ret)
 822                 return ret;
 823         if (!dev_is_sata(dev))
 824                 sas_change_queue_depth(sdev, 64);
 825
 826         return 0;
 827 }
 828 EXPORT_SYMBOL_GPL(hisi_sas_slave_configure);
 829
 830 void hisi_sas_scan_start(struct Scsi_Host *shost)
 831 {
 832         struct hisi_hba *hisi_hba = shost_priv(shost);
 833
 834         hisi_hba->hw->phys_init(hisi_hba);
 835 }
 836 EXPORT_SYMBOL_GPL(hisi_sas_scan_start);
 837
 838 int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
 839 {
 840         struct hisi_hba *hisi_hba = shost_priv(shost);
 841         struct sas_ha_struct *sha = &hisi_hba->sha;
 842
 843         /* Wait for PHY up interrupt to occur */
 844         if (time < HZ)
 845                 return 0;
 846
 847         sas_drain_work(sha);
 848         return 1;
 849 }
 850 EXPORT_SYMBOL_GPL(hisi_sas_scan_finished);
 851
 852 static void hisi_sas_phyup_work(struct work_struct *work)
 853 {
 854         struct hisi_sas_phy *phy =
 855                 container_of(work, typeof(*phy), works[HISI_PHYE_PHY_UP]);
 856         struct hisi_hba *hisi_hba = phy->hisi_hba;
 857         struct asd_sas_phy *sas_phy = &phy->sas_phy;
 858         int phy_no = sas_phy->id;
 859
 860         if (phy->identify.target_port_protocols == SAS_PROTOCOL_SSP)
 861                 hisi_hba->hw->sl_notify_ssp(hisi_hba, phy_no);
 862         hisi_sas_bytes_dmaed(hisi_hba, phy_no);
 863 }
 864
 865 static void hisi_sas_linkreset_work(struct work_struct *work)
 866 {
 867         struct hisi_sas_phy *phy =
 868                 container_of(work, typeof(*phy), works[HISI_PHYE_LINK_RESET]);
 869         struct asd_sas_phy *sas_phy = &phy->sas_phy;
 870
 871         hisi_sas_control_phy(sas_phy, PHY_FUNC_LINK_RESET, NULL);
 872 }
 873
 874 static const work_func_t hisi_sas_phye_fns[HISI_PHYES_NUM] = {
 875         [HISI_PHYE_PHY_UP] = hisi_sas_phyup_work,
 876         [HISI_PHYE_LINK_RESET] = hisi_sas_linkreset_work,
 877 };
 878
 879 bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy,
 880                                 enum hisi_sas_phy_event event)
 881 {
 882         struct hisi_hba *hisi_hba = phy->hisi_hba;
 883
 884         if (WARN_ON(event >= HISI_PHYES_NUM))
 885                 return false;
 886
 887         return queue_work(hisi_hba->wq, &phy->works[event]);
 888 }
 889 EXPORT_SYMBOL_GPL(hisi_sas_notify_phy_event);
 890
 891 static void hisi_sas_wait_phyup_timedout(struct timer_list *t)
 892 {
 893         struct hisi_sas_phy *phy = from_timer(phy, t, timer);
 894         struct hisi_hba *hisi_hba = phy->hisi_hba;
 895         struct device *dev = hisi_hba->dev;
 896         int phy_no = phy->sas_phy.id;
 897
 898         dev_warn(dev, "phy%d wait phyup timeout, issuing link reset\n", phy_no);
 899         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
 900 }
 901
 902 void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no)
 903 {
 904         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 905         struct device *dev = hisi_hba->dev;
 906
 907         dev_dbg(dev, "phy%d OOB ready\n", phy_no);
 908         if (phy->phy_attached)
 909                 return;
 910
 911         if (!timer_pending(&phy->timer)) {
 912                 phy->timer.expires = jiffies + HISI_SAS_WAIT_PHYUP_TIMEOUT * HZ;
 913                 add_timer(&phy->timer);
 914         }
 915 }
 916 EXPORT_SYMBOL_GPL(hisi_sas_phy_oob_ready);
 917
 918 static void hisi_sas_phy_init(struct hisi_hba *hisi_hba, int phy_no)
 919 {
 920         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 921         struct asd_sas_phy *sas_phy = &phy->sas_phy;
 922         int i;
 923
 924         phy->hisi_hba = hisi_hba;
 925         phy->port = NULL;
 926         phy->minimum_linkrate = SAS_LINK_RATE_1_5_GBPS;
 927         phy->maximum_linkrate = hisi_hba->hw->phy_get_max_linkrate();
 928         sas_phy->enabled = (phy_no < hisi_hba->n_phy) ? 1 : 0;
 929         sas_phy->class = SAS;
 930         sas_phy->iproto = SAS_PROTOCOL_ALL;
 931         sas_phy->tproto = 0;
 932         sas_phy->type = PHY_TYPE_PHYSICAL;
 933         sas_phy->role = PHY_ROLE_INITIATOR;
 934         sas_phy->oob_mode = OOB_NOT_CONNECTED;
 935         sas_phy->linkrate = SAS_LINK_RATE_UNKNOWN;
 936         sas_phy->id = phy_no;
 937         sas_phy->sas_addr = &hisi_hba->sas_addr[0];
 938         sas_phy->frame_rcvd = &phy->frame_rcvd[0];
 939         sas_phy->ha = (struct sas_ha_struct *)hisi_hba->shost->hostdata;
 940         sas_phy->lldd_phy = phy;
 941
 942         for (i = 0; i < HISI_PHYES_NUM; i++)
 943                 INIT_WORK(&phy->works[i], hisi_sas_phye_fns[i]);
 944
 945         spin_lock_init(&phy->lock);
 946
 947         timer_setup(&phy->timer, hisi_sas_wait_phyup_timedout, 0);
 948 }
 949
 950 /* Wrapper to ensure we track hisi_sas_phy.enable properly */
 951 void hisi_sas_phy_enable(struct hisi_hba *hisi_hba, int phy_no, int enable)
 952 {
 953         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 954         struct asd_sas_phy *aphy = &phy->sas_phy;
 955         struct sas_phy *sphy = aphy->phy;
 956         unsigned long flags;
 957
 958         spin_lock_irqsave(&phy->lock, flags);
 959
 960         if (enable) {
 961                 /* We may have been enabled already; if so, don't touch */
 962                 if (!phy->enable)
 963                         sphy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
 964                 hisi_hba->hw->phy_start(hisi_hba, phy_no);
 965         } else {
 966                 sphy->negotiated_linkrate = SAS_PHY_DISABLED;
 967                 hisi_hba->hw->phy_disable(hisi_hba, phy_no);
 968         }
 969         phy->enable = enable;
 970         spin_unlock_irqrestore(&phy->lock, flags);
 971 }
 972 EXPORT_SYMBOL_GPL(hisi_sas_phy_enable);
 973
 974 static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy)
 975 {
 976         struct sas_ha_struct *sas_ha = sas_phy->ha;
 977         struct hisi_hba *hisi_hba = sas_ha->lldd_ha;
 978         struct hisi_sas_phy *phy = sas_phy->lldd_phy;
 979         struct asd_sas_port *sas_port = sas_phy->port;
 980         struct hisi_sas_port *port;
 981         unsigned long flags;
 982
 983         if (!sas_port)
 984                 return;
 985
 986         port = to_hisi_sas_port(sas_port);
 987         spin_lock_irqsave(&hisi_hba->lock, flags);
 988         port->port_attached = 1;
 989         port->id = phy->port_id;
 990         phy->port = port;
 991         sas_port->lldd_port = port;
 992         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 993 }
 994
 995 static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task *task,
 996                                      struct hisi_sas_slot *slot)
 997 {
 998         if (task) {
 999                 unsigned long flags;
1000                 struct task_status_struct *ts;
1001
1002                 ts = &task->task_status;
1003
1004                 ts->resp = SAS_TASK_COMPLETE;
1005                 ts->stat = SAS_ABORTED_TASK;
1006                 spin_lock_irqsave(&task->task_state_lock, flags);
1007                 task->task_state_flags &=
1008                         ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1009                 if (!slot->is_internal && task->task_proto != SAS_PROTOCOL_SMP)
1010                         task->task_state_flags |= SAS_TASK_STATE_DONE;
1011                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1012         }
1013
1014         hisi_sas_slot_task_free(hisi_hba, task, slot);
1015 }
1016
1017 static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
1018                         struct domain_device *device)
1019 {
1020         struct hisi_sas_slot *slot, *slot2;
1021         struct hisi_sas_device *sas_dev = device->lldd_dev;
1022
1023         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry)
1024                 hisi_sas_do_release_task(hisi_hba, slot->task, slot);
1025 }
1026
1027 void hisi_sas_release_tasks(struct hisi_hba *hisi_hba)
1028 {
1029         struct hisi_sas_device *sas_dev;
1030         struct domain_device *device;
1031         int i;
1032
1033         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
1034                 sas_dev = &hisi_hba->devices[i];
1035                 device = sas_dev->sas_device;
1036
1037                 if ((sas_dev->dev_type == SAS_PHY_UNUSED) ||
1038                     !device)
1039                         continue;
1040
1041                 hisi_sas_release_task(hisi_hba, device);
1042         }
1043 }
1044 EXPORT_SYMBOL_GPL(hisi_sas_release_tasks);
1045
1046 static void hisi_sas_dereg_device(struct hisi_hba *hisi_hba,
1047                                 struct domain_device *device)
1048 {
1049         if (hisi_hba->hw->dereg_device)
1050                 hisi_hba->hw->dereg_device(hisi_hba, device);
1051 }
1052
1053 static void hisi_sas_dev_gone(struct domain_device *device)
1054 {
1055         struct hisi_sas_device *sas_dev = device->lldd_dev;
1056         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1057         struct device *dev = hisi_hba->dev;
1058
1059         dev_info(dev, "dev[%d:%x] is gone\n",
1060                  sas_dev->device_id, sas_dev->dev_type);
1061
1062         down(&hisi_hba->sem);
1063         if (!test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) {
1064                 hisi_sas_internal_task_abort(hisi_hba, device,
1065                                              HISI_SAS_INT_ABT_DEV, 0);
1066
1067                 hisi_sas_dereg_device(hisi_hba, device);
1068
1069                 hisi_hba->hw->clear_itct(hisi_hba, sas_dev);
1070                 device->lldd_dev = NULL;
1071         }
1072
1073         if (hisi_hba->hw->free_device)
1074                 hisi_hba->hw->free_device(sas_dev);
1075         sas_dev->dev_type = SAS_PHY_UNUSED;
1076         sas_dev->sas_device = NULL;
1077         up(&hisi_hba->sem);
1078 }
1079
1080 static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags)
1081 {
1082         return hisi_sas_task_exec(task, gfp_flags, 0, NULL);
1083 }
1084
1085 static int hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no,
1086                         struct sas_phy_linkrates *r)
1087 {
1088         struct sas_phy_linkrates _r;
1089
1090         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1091         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1092         enum sas_linkrate min, max;
1093
1094         if (r->minimum_linkrate > SAS_LINK_RATE_1_5_GBPS)
1095                 return -EINVAL;
1096
1097         if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1098                 max = sas_phy->phy->maximum_linkrate;
1099                 min = r->minimum_linkrate;
1100         } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1101                 max = r->maximum_linkrate;
1102                 min = sas_phy->phy->minimum_linkrate;
1103         } else
1104                 return -EINVAL;
1105
1106         _r.maximum_linkrate = max;
1107         _r.minimum_linkrate = min;
1108
1109         sas_phy->phy->maximum_linkrate = max;
1110         sas_phy->phy->minimum_linkrate = min;
1111
1112         hisi_sas_phy_enable(hisi_hba, phy_no, 0);
1113         msleep(100);
1114         hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, &_r);
1115         hisi_sas_phy_enable(hisi_hba, phy_no, 1);
1116
1117         return 0;
1118 }
1119
1120 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
1121                                 void *funcdata)
1122 {
1123         struct sas_ha_struct *sas_ha = sas_phy->ha;
1124         struct hisi_hba *hisi_hba = sas_ha->lldd_ha;
1125         int phy_no = sas_phy->id;
1126
1127         switch (func) {
1128         case PHY_FUNC_HARD_RESET:
1129                 hisi_hba->hw->phy_hard_reset(hisi_hba, phy_no);
1130                 break;
1131
1132         case PHY_FUNC_LINK_RESET:
1133                 hisi_sas_phy_enable(hisi_hba, phy_no, 0);
1134                 msleep(100);
1135                 hisi_sas_phy_enable(hisi_hba, phy_no, 1);
1136                 break;
1137
1138         case PHY_FUNC_DISABLE:
1139                 hisi_sas_phy_enable(hisi_hba, phy_no, 0);
1140                 break;
1141
1142         case PHY_FUNC_SET_LINK_RATE:
1143                 return hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata);
1144         case PHY_FUNC_GET_EVENTS:
1145                 if (hisi_hba->hw->get_events) {
1146                         hisi_hba->hw->get_events(hisi_hba, phy_no);
1147                         break;
1148                 }
1149                 /* fallthru */
1150         case PHY_FUNC_RELEASE_SPINUP_HOLD:
1151         default:
1152                 return -EOPNOTSUPP;
1153         }
1154         return 0;
1155 }
1156
1157 static void hisi_sas_task_done(struct sas_task *task)
1158 {
1159         del_timer(&task->slow_task->timer);
1160         complete(&task->slow_task->completion);
1161 }
1162
1163 static void hisi_sas_tmf_timedout(struct timer_list *t)
1164 {
1165         struct sas_task_slow *slow = from_timer(slow, t, timer);
1166         struct sas_task *task = slow->task;
1167         unsigned long flags;
1168         bool is_completed = true;
1169
1170         spin_lock_irqsave(&task->task_state_lock, flags);
1171         if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
1172                 task->task_state_flags |= SAS_TASK_STATE_ABORTED;
1173                 is_completed = false;
1174         }
1175         spin_unlock_irqrestore(&task->task_state_lock, flags);
1176
1177         if (!is_completed)
1178                 complete(&task->slow_task->completion);
1179 }
1180
1181 #define TASK_TIMEOUT 20
1182 #define TASK_RETRY 3
1183 #define INTERNAL_ABORT_TIMEOUT 6
1184 static int hisi_sas_exec_internal_tmf_task(struct domain_device *device,
1185                                            void *parameter, u32 para_len,
1186                                            struct hisi_sas_tmf_task *tmf)
1187 {
1188         struct hisi_sas_device *sas_dev = device->lldd_dev;
1189         struct hisi_hba *hisi_hba = sas_dev->hisi_hba;
1190         struct device *dev = hisi_hba->dev;
1191         struct sas_task *task;
1192         int res, retry;
1193
1194         for (retry = 0; retry < TASK_RETRY; retry++) {
1195                 task = sas_alloc_slow_task(GFP_KERNEL);
1196                 if (!task)
1197                         return -ENOMEM;
1198
1199                 task->dev = device;
1200                 task->task_proto = device->tproto;
1201
1202                 if (dev_is_sata(device)) {
1203                         task->ata_task.device_control_reg_update = 1;
1204                         memcpy(&task->ata_task.fis, parameter, para_len);
1205                 } else {
1206                         memcpy(&task->ssp_task, parameter, para_len);
1207                 }
1208                 task->task_done = hisi_sas_task_done;
1209
1210                 task->slow_task->timer.function = hisi_sas_tmf_timedout;
1211                 task->slow_task->timer.expires = jiffies + TASK_TIMEOUT * HZ;
1212                 add_timer(&task->slow_task->timer);
1213
1214                 res = hisi_sas_task_exec(task, GFP_KERNEL, 1, tmf);
1215
1216                 if (res) {
1217                         del_timer(&task->slow_task->timer);
1218                         dev_err(dev, "abort tmf: executing internal task failed: %d\n",
1219                                 res);
1220                         goto ex_err;
1221                 }
1222
1223                 wait_for_completion(&task->slow_task->completion);
1224                 res = TMF_RESP_FUNC_FAILED;
1225                 /* Even TMF timed out, return direct. */
1226                 if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) {
1227                         if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
1228                                 struct hisi_sas_slot *slot = task->lldd_task;
1229
1230                                 dev_err(dev, "abort tmf: TMF task timeout and not done\n");
1231                                 if (slot) {
1232                                         struct hisi_sas_cq *cq =
1233                                                &hisi_hba->cq[slot->dlvry_queue];
1234                                         /*
1235                                          * flush tasklet to avoid free'ing task
1236                                          * before using task in IO completion
1237                                          */
1238                                         tasklet_kill(&cq->tasklet);
1239                                         slot->task = NULL;
1240                                 }
1241
1242                                 goto ex_err;
1243                         } else
1244                                 dev_err(dev, "abort tmf: TMF task timeout\n");
1245                 }
1246
1247                 if (task->task_status.resp == SAS_TASK_COMPLETE &&
1248                      task->task_status.stat == TMF_RESP_FUNC_COMPLETE) {
1249                         res = TMF_RESP_FUNC_COMPLETE;
1250                         break;
1251                 }
1252
1253                 if (task->task_status.resp == SAS_TASK_COMPLETE &&
1254                         task->task_status.stat == TMF_RESP_FUNC_SUCC) {
1255                         res = TMF_RESP_FUNC_SUCC;
1256                         break;
1257                 }
1258
1259                 if (task->task_status.resp == SAS_TASK_COMPLETE &&
1260                       task->task_status.stat == SAS_DATA_UNDERRUN) {
1261                         /* no error, but return the number of bytes of
1262                          * underrun
1263                          */
1264                         dev_warn(dev, "abort tmf: task to dev %016llx resp: 0x%x sts 0x%x underrun\n",
1265                                  SAS_ADDR(device->sas_addr),
1266                                  task->task_status.resp,
1267                                  task->task_status.stat);
1268                         res = task->task_status.residual;
1269                         break;
1270                 }
1271
1272                 if (task->task_status.resp == SAS_TASK_COMPLETE &&
1273                         task->task_status.stat == SAS_DATA_OVERRUN) {
1274                         dev_warn(dev, "abort tmf: blocked task error\n");
1275                         res = -EMSGSIZE;
1276                         break;
1277                 }
1278
1279                 if (task->task_status.resp == SAS_TASK_COMPLETE &&
1280                     task->task_status.stat == SAS_OPEN_REJECT) {
1281                         dev_warn(dev, "abort tmf: open reject failed\n");
1282                         res = -EIO;
1283                 } else {
1284                         dev_warn(dev, "abort tmf: task to dev %016llx resp: 0x%x status 0x%x\n",
1285                                  SAS_ADDR(device->sas_addr),
1286                                  task->task_status.resp,
1287                                  task->task_status.stat);
1288                 }
1289                 sas_free_task(task);
1290                 task = NULL;
1291         }
1292 ex_err:
1293         if (retry == TASK_RETRY)
1294                 dev_warn(dev, "abort tmf: executing internal task failed!\n");
1295         sas_free_task(task);
1296         return res;
1297 }
1298
1299 static void hisi_sas_fill_ata_reset_cmd(struct ata_device *dev,
1300                 bool reset, int pmp, u8 *fis)
1301 {
1302         struct ata_taskfile tf;
1303
1304         ata_tf_init(dev, &tf);
1305         if (reset)
1306                 tf.ctl |= ATA_SRST;
1307         else
1308                 tf.ctl &= ~ATA_SRST;
1309         tf.command = ATA_CMD_DEV_RESET;
1310         ata_tf_to_fis(&tf, pmp, 0, fis);
1311 }
1312
1313 static int hisi_sas_softreset_ata_disk(struct domain_device *device)
1314 {
1315         u8 fis[20] = {0};
1316         struct ata_port *ap = device->sata_dev.ap;
1317         struct ata_link *link;
1318         int rc = TMF_RESP_FUNC_FAILED;
1319         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1320         struct device *dev = hisi_hba->dev;
1321         int s = sizeof(struct host_to_dev_fis);
1322
1323         ata_for_each_link(link, ap, EDGE) {
1324                 int pmp = sata_srst_pmp(link);
1325
1326                 hisi_sas_fill_ata_reset_cmd(link->device, 1, pmp, fis);
1327                 rc = hisi_sas_exec_internal_tmf_task(device, fis, s, NULL);
1328                 if (rc != TMF_RESP_FUNC_COMPLETE)
1329                         break;
1330         }
1331
1332         if (rc == TMF_RESP_FUNC_COMPLETE) {
1333                 ata_for_each_link(link, ap, EDGE) {
1334                         int pmp = sata_srst_pmp(link);
1335
1336                         hisi_sas_fill_ata_reset_cmd(link->device, 0, pmp, fis);
1337                         rc = hisi_sas_exec_internal_tmf_task(device, fis,
1338                                                              s, NULL);
1339                         if (rc != TMF_RESP_FUNC_COMPLETE)
1340                                 dev_err(dev, "ata disk de-reset failed\n");
1341                 }
1342         } else {
1343                 dev_err(dev, "ata disk reset failed\n");
1344         }
1345
1346         if (rc == TMF_RESP_FUNC_COMPLETE)
1347                 hisi_sas_release_task(hisi_hba, device);
1348
1349         return rc;
1350 }
1351
1352 static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device,
1353                                 u8 *lun, struct hisi_sas_tmf_task *tmf)
1354 {
1355         struct sas_ssp_task ssp_task;
1356
1357         if (!(device->tproto & SAS_PROTOCOL_SSP))
1358                 return TMF_RESP_FUNC_ESUPP;
1359
1360         memcpy(ssp_task.LUN, lun, 8);
1361
1362         return hisi_sas_exec_internal_tmf_task(device, &ssp_task,
1363                                 sizeof(ssp_task), tmf);
1364 }
1365
1366 static void hisi_sas_refresh_port_id(struct hisi_hba *hisi_hba)
1367 {
1368         u32 state = hisi_hba->hw->get_phys_state(hisi_hba);
1369         int i;
1370
1371         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
1372                 struct hisi_sas_device *sas_dev = &hisi_hba->devices[i];
1373                 struct domain_device *device = sas_dev->sas_device;
1374                 struct asd_sas_port *sas_port;
1375                 struct hisi_sas_port *port;
1376                 struct hisi_sas_phy *phy = NULL;
1377                 struct asd_sas_phy *sas_phy;
1378
1379                 if ((sas_dev->dev_type == SAS_PHY_UNUSED)
1380                                 || !device || !device->port)
1381                         continue;
1382
1383                 sas_port = device->port;
1384                 port = to_hisi_sas_port(sas_port);
1385
1386                 list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el)
1387                         if (state & BIT(sas_phy->id)) {
1388                                 phy = sas_phy->lldd_phy;
1389                                 break;
1390                         }
1391
1392                 if (phy) {
1393                         port->id = phy->port_id;
1394
1395                         /* Update linkrate of directly attached device. */
1396                         if (!device->parent)
1397                                 device->linkrate = phy->sas_phy.linkrate;
1398
1399                         hisi_hba->hw->setup_itct(hisi_hba, sas_dev);
1400                 } else
1401                         port->id = 0xff;
1402         }
1403 }
1404
1405 static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state)
1406 {
1407         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1408         struct asd_sas_port *_sas_port = NULL;
1409         int phy_no;
1410
1411         for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
1412                 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1413                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1414                 struct asd_sas_port *sas_port = sas_phy->port;
1415                 bool do_port_check = !!(_sas_port != sas_port);
1416
1417                 if (!sas_phy->phy->enabled)
1418                         continue;
1419
1420                 /* Report PHY state change to libsas */
1421                 if (state & BIT(phy_no)) {
1422                         if (do_port_check && sas_port && sas_port->port_dev) {
1423                                 struct domain_device *dev = sas_port->port_dev;
1424
1425                                 _sas_port = sas_port;
1426
1427                                 if (dev_is_expander(dev->dev_type))
1428                                         sas_ha->notify_port_event(sas_phy,
1429                                                         PORTE_BROADCAST_RCVD);
1430                         }
1431                 } else {
1432                         hisi_sas_phy_down(hisi_hba, phy_no, 0);
1433                 }
1434
1435         }
1436 }
1437
1438 static void hisi_sas_reset_init_all_devices(struct hisi_hba *hisi_hba)
1439 {
1440         struct hisi_sas_device *sas_dev;
1441         struct domain_device *device;
1442         int i;
1443
1444         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
1445                 sas_dev = &hisi_hba->devices[i];
1446                 device = sas_dev->sas_device;
1447
1448                 if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
1449                         continue;
1450
1451                 hisi_sas_init_device(device);
1452         }
1453 }
1454
1455 static void hisi_sas_send_ata_reset_each_phy(struct hisi_hba *hisi_hba,
1456                                              struct asd_sas_port *sas_port,
1457                                              struct domain_device *device)
1458 {
1459         struct hisi_sas_tmf_task tmf_task = { .force_phy = 1 };
1460         struct ata_port *ap = device->sata_dev.ap;
1461         struct device *dev = hisi_hba->dev;
1462         int s = sizeof(struct host_to_dev_fis);
1463         int rc = TMF_RESP_FUNC_FAILED;
1464         struct asd_sas_phy *sas_phy;
1465         struct ata_link *link;
1466         u8 fis[20] = {0};
1467         u32 state;
1468
1469         state = hisi_hba->hw->get_phys_state(hisi_hba);
1470         list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el) {
1471                 if (!(state & BIT(sas_phy->id)))
1472                         continue;
1473
1474                 ata_for_each_link(link, ap, EDGE) {
1475                         int pmp = sata_srst_pmp(link);
1476
1477                         tmf_task.phy_id = sas_phy->id;
1478                         hisi_sas_fill_ata_reset_cmd(link->device, 1, pmp, fis);
1479                         rc = hisi_sas_exec_internal_tmf_task(device, fis, s,
1480                                                              &tmf_task);
1481                         if (rc != TMF_RESP_FUNC_COMPLETE) {
1482                                 dev_err(dev, "phy%d ata reset failed rc=%d\n",
1483                                         sas_phy->id, rc);
1484                                 break;
1485                         }
1486                 }
1487         }
1488 }
1489
1490 static void hisi_sas_terminate_stp_reject(struct hisi_hba *hisi_hba)
1491 {
1492         struct device *dev = hisi_hba->dev;
1493         int port_no, rc, i;
1494
1495         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
1496                 struct hisi_sas_device *sas_dev = &hisi_hba->devices[i];
1497                 struct domain_device *device = sas_dev->sas_device;
1498
1499                 if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
1500                         continue;
1501
1502                 rc = hisi_sas_internal_task_abort(hisi_hba, device,
1503                                                   HISI_SAS_INT_ABT_DEV, 0);
1504                 if (rc < 0)
1505                         dev_err(dev, "STP reject: abort dev failed %d\n", rc);
1506         }
1507
1508         for (port_no = 0; port_no < hisi_hba->n_phy; port_no++) {
1509                 struct hisi_sas_port *port = &hisi_hba->port[port_no];
1510                 struct asd_sas_port *sas_port = &port->sas_port;
1511                 struct domain_device *port_dev = sas_port->port_dev;
1512                 struct domain_device *device;
1513
1514                 if (!port_dev || !dev_is_expander(port_dev->dev_type))
1515                         continue;
1516
1517                 /* Try to find a SATA device */
1518                 list_for_each_entry(device, &sas_port->dev_list,
1519                                     dev_list_node) {
1520                         if (dev_is_sata(device)) {
1521                                 hisi_sas_send_ata_reset_each_phy(hisi_hba,
1522                                                                  sas_port,
1523                                                                  device);
1524                                 break;
1525                         }
1526                 }
1527         }
1528 }
1529
1530 void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba)
1531 {
1532         struct Scsi_Host *shost = hisi_hba->shost;
1533
1534         down(&hisi_hba->sem);
1535         hisi_hba->phy_state = hisi_hba->hw->get_phys_state(hisi_hba);
1536
1537         scsi_block_requests(shost);
1538         hisi_hba->hw->wait_cmds_complete_timeout(hisi_hba, 100, 5000);
1539
1540         if (timer_pending(&hisi_hba->timer))
1541                 del_timer_sync(&hisi_hba->timer);
1542
1543         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
1544 }
1545 EXPORT_SYMBOL_GPL(hisi_sas_controller_reset_prepare);
1546
1547 void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba)
1548 {
1549         struct Scsi_Host *shost = hisi_hba->shost;
1550         u32 state;
1551
1552         /* Init and wait for PHYs to come up and all libsas event finished. */
1553         hisi_hba->hw->phys_init(hisi_hba);
1554         msleep(1000);
1555         hisi_sas_refresh_port_id(hisi_hba);
1556         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
1557
1558         if (hisi_hba->reject_stp_links_msk)
1559                 hisi_sas_terminate_stp_reject(hisi_hba);
1560         hisi_sas_reset_init_all_devices(hisi_hba);
1561         up(&hisi_hba->sem);
1562         scsi_unblock_requests(shost);
1563         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
1564
1565         state = hisi_hba->hw->get_phys_state(hisi_hba);
1566         hisi_sas_rescan_topology(hisi_hba, state);
1567 }
1568 EXPORT_SYMBOL_GPL(hisi_sas_controller_reset_done);
1569
1570 static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
1571 {
1572         struct device *dev = hisi_hba->dev;
1573         struct Scsi_Host *shost = hisi_hba->shost;
1574         int rc;
1575
1576         if (hisi_sas_debugfs_enable && hisi_hba->debugfs_itct)
1577                 queue_work(hisi_hba->wq, &hisi_hba->debugfs_work);
1578
1579         if (!hisi_hba->hw->soft_reset)
1580                 return -1;
1581
1582         if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
1583                 return -1;
1584
1585         dev_info(dev, "controller resetting...\n");
1586         hisi_sas_controller_reset_prepare(hisi_hba);
1587
1588         rc = hisi_hba->hw->soft_reset(hisi_hba);
1589         if (rc) {
1590                 dev_warn(dev, "controller reset failed (%d)\n", rc);
1591                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
1592                 up(&hisi_hba->sem);
1593                 scsi_unblock_requests(shost);
1594                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
1595                 return rc;
1596         }
1597
1598         hisi_sas_controller_reset_done(hisi_hba);
1599         dev_info(dev, "controller reset complete\n");
1600
1601         return 0;
1602 }
1603
1604 static int hisi_sas_abort_task(struct sas_task *task)
1605 {
1606         struct scsi_lun lun;
1607         struct hisi_sas_tmf_task tmf_task;
1608         struct domain_device *device = task->dev;
1609         struct hisi_sas_device *sas_dev = device->lldd_dev;
1610         struct hisi_hba *hisi_hba;
1611         struct device *dev;
1612         int rc = TMF_RESP_FUNC_FAILED;
1613         unsigned long flags;
1614
1615         if (!sas_dev)
1616                 return TMF_RESP_FUNC_FAILED;
1617
1618         hisi_hba = dev_to_hisi_hba(task->dev);
1619         dev = hisi_hba->dev;
1620
1621         spin_lock_irqsave(&task->task_state_lock, flags);
1622         if (task->task_state_flags & SAS_TASK_STATE_DONE) {
1623                 struct hisi_sas_slot *slot = task->lldd_task;
1624                 struct hisi_sas_cq *cq;
1625
1626                 if (slot) {
1627                         /*
1628                          * flush tasklet to avoid free'ing task
1629                          * before using task in IO completion
1630                          */
1631                         cq = &hisi_hba->cq[slot->dlvry_queue];
1632                         tasklet_kill(&cq->tasklet);
1633                 }
1634                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1635                 rc = TMF_RESP_FUNC_COMPLETE;
1636                 goto out;
1637         }
1638         task->task_state_flags |= SAS_TASK_STATE_ABORTED;
1639         spin_unlock_irqrestore(&task->task_state_lock, flags);
1640
1641         if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) {
1642                 struct scsi_cmnd *cmnd = task->uldd_task;
1643                 struct hisi_sas_slot *slot = task->lldd_task;
1644                 u16 tag = slot->idx;
1645                 int rc2;
1646
1647                 int_to_scsilun(cmnd->device->lun, &lun);
1648                 tmf_task.tmf = TMF_ABORT_TASK;
1649                 tmf_task.tag_of_task_to_be_managed = tag;
1650
1651                 rc = hisi_sas_debug_issue_ssp_tmf(task->dev, lun.scsi_lun,
1652                                                   &tmf_task);
1653
1654                 rc2 = hisi_sas_internal_task_abort(hisi_hba, device,
1655                                                    HISI_SAS_INT_ABT_CMD, tag);
1656                 if (rc2 < 0) {
1657                         dev_err(dev, "abort task: internal abort (%d)\n", rc2);
1658                         return TMF_RESP_FUNC_FAILED;
1659                 }
1660
1661                 /*
1662                  * If the TMF finds that the IO is not in the device and also
1663                  * the internal abort does not succeed, then it is safe to
1664                  * free the slot.
1665                  * Note: if the internal abort succeeds then the slot
1666                  * will have already been completed
1667                  */
1668                 if (rc == TMF_RESP_FUNC_COMPLETE && rc2 != TMF_RESP_FUNC_SUCC) {
1669                         if (task->lldd_task)
1670                                 hisi_sas_do_release_task(hisi_hba, task, slot);
1671                 }
1672         } else if (task->task_proto & SAS_PROTOCOL_SATA ||
1673                 task->task_proto & SAS_PROTOCOL_STP) {
1674                 if (task->dev->dev_type == SAS_SATA_DEV) {
1675                         rc = hisi_sas_internal_task_abort(hisi_hba, device,
1676                                                           HISI_SAS_INT_ABT_DEV,
1677                                                           0);
1678                         if (rc < 0) {
1679                                 dev_err(dev, "abort task: internal abort failed\n");
1680                                 goto out;
1681                         }
1682                         hisi_sas_dereg_device(hisi_hba, device);
1683                         rc = hisi_sas_softreset_ata_disk(device);
1684                 }
1685         } else if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SMP) {
1686                 /* SMP */
1687                 struct hisi_sas_slot *slot = task->lldd_task;
1688                 u32 tag = slot->idx;
1689                 struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue];
1690
1691                 rc = hisi_sas_internal_task_abort(hisi_hba, device,
1692                                                   HISI_SAS_INT_ABT_CMD, tag);
1693                 if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) &&
1694                                         task->lldd_task) {
1695                         /*
1696                          * flush tasklet to avoid free'ing task
1697                          * before using task in IO completion
1698                          */
1699                         tasklet_kill(&cq->tasklet);
1700                         slot->task = NULL;
1701                 }
1702         }
1703
1704 out:
1705         if (rc != TMF_RESP_FUNC_COMPLETE)
1706                 dev_notice(dev, "abort task: rc=%d\n", rc);
1707         return rc;
1708 }
1709
1710 static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
1711 {
1712         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1713         struct device *dev = hisi_hba->dev;
1714         struct hisi_sas_tmf_task tmf_task;
1715         int rc;
1716
1717         rc = hisi_sas_internal_task_abort(hisi_hba, device,
1718                                           HISI_SAS_INT_ABT_DEV, 0);
1719         if (rc < 0) {
1720                 dev_err(dev, "abort task set: internal abort rc=%d\n", rc);
1721                 return TMF_RESP_FUNC_FAILED;
1722         }
1723         hisi_sas_dereg_device(hisi_hba, device);
1724
1725         tmf_task.tmf = TMF_ABORT_TASK_SET;
1726         rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
1727
1728         if (rc == TMF_RESP_FUNC_COMPLETE)
1729                 hisi_sas_release_task(hisi_hba, device);
1730
1731         return rc;
1732 }
1733
1734 static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun)
1735 {
1736         struct hisi_sas_tmf_task tmf_task;
1737         int rc;
1738
1739         tmf_task.tmf = TMF_CLEAR_ACA;
1740         rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
1741
1742         return rc;
1743 }
1744
1745 static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device)
1746 {
1747         struct sas_phy *local_phy = sas_get_local_phy(device);
1748         struct hisi_sas_device *sas_dev = device->lldd_dev;
1749         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1750         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1751         DECLARE_COMPLETION_ONSTACK(phyreset);
1752         int rc, reset_type;
1753
1754         if (!local_phy->enabled) {
1755                 sas_put_local_phy(local_phy);
1756                 return -ENODEV;
1757         }
1758
1759         if (scsi_is_sas_phy_local(local_phy)) {
1760                 struct asd_sas_phy *sas_phy =
1761                         sas_ha->sas_phy[local_phy->number];
1762                 struct hisi_sas_phy *phy =
1763                         container_of(sas_phy, struct hisi_sas_phy, sas_phy);
1764                 phy->in_reset = 1;
1765                 phy->reset_completion = &phyreset;
1766         }
1767
1768         reset_type = (sas_dev->dev_status == HISI_SAS_DEV_INIT ||
1769                       !dev_is_sata(device)) ? true : false;
1770
1771         rc = sas_phy_reset(local_phy, reset_type);
1772         sas_put_local_phy(local_phy);
1773
1774         if (scsi_is_sas_phy_local(local_phy)) {
1775                 struct asd_sas_phy *sas_phy =
1776                         sas_ha->sas_phy[local_phy->number];
1777                 struct hisi_sas_phy *phy =
1778                         container_of(sas_phy, struct hisi_sas_phy, sas_phy);
1779                 int ret = wait_for_completion_timeout(&phyreset, 2 * HZ);
1780                 unsigned long flags;
1781
1782                 spin_lock_irqsave(&phy->lock, flags);
1783                 phy->reset_completion = NULL;
1784                 phy->in_reset = 0;
1785                 spin_unlock_irqrestore(&phy->lock, flags);
1786
1787                 /* report PHY down if timed out */
1788                 if (!ret)
1789                         hisi_sas_phy_down(hisi_hba, sas_phy->id, 0);
1790         } else if (sas_dev->dev_status != HISI_SAS_DEV_INIT) {
1791                 /*
1792                  * If in init state, we rely on caller to wait for link to be
1793                  * ready; otherwise, except phy reset is fail, delay.
1794                  */
1795                 if (!rc)
1796                         msleep(2000);
1797         }
1798
1799         return rc;
1800 }
1801
1802 static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
1803 {
1804         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1805         struct device *dev = hisi_hba->dev;
1806         int rc;
1807
1808         rc = hisi_sas_internal_task_abort(hisi_hba, device,
1809                                           HISI_SAS_INT_ABT_DEV, 0);
1810         if (rc < 0) {
1811                 dev_err(dev, "I_T nexus reset: internal abort (%d)\n", rc);
1812                 return TMF_RESP_FUNC_FAILED;
1813         }
1814         hisi_sas_dereg_device(hisi_hba, device);
1815
1816         if (dev_is_sata(device)) {
1817                 rc = hisi_sas_softreset_ata_disk(device);
1818                 if (rc == TMF_RESP_FUNC_FAILED)
1819                         return TMF_RESP_FUNC_FAILED;
1820         }
1821
1822         rc = hisi_sas_debug_I_T_nexus_reset(device);
1823
1824         if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV))
1825                 hisi_sas_release_task(hisi_hba, device);
1826
1827         return rc;
1828 }
1829
1830 static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
1831 {
1832         struct hisi_sas_device *sas_dev = device->lldd_dev;
1833         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
1834         struct device *dev = hisi_hba->dev;
1835         int rc = TMF_RESP_FUNC_FAILED;
1836
1837         /* Clear internal IO and then lu reset */
1838         rc = hisi_sas_internal_task_abort(hisi_hba, device,
1839                                           HISI_SAS_INT_ABT_DEV, 0);
1840         if (rc < 0) {
1841                 dev_err(dev, "lu_reset: internal abort failed\n");
1842                 goto out;
1843         }
1844         hisi_sas_dereg_device(hisi_hba, device);
1845
1846         if (dev_is_sata(device)) {
1847                 struct sas_phy *phy;
1848
1849                 phy = sas_get_local_phy(device);
1850
1851                 rc = sas_phy_reset(phy, true);
1852
1853                 if (rc == 0)
1854                         hisi_sas_release_task(hisi_hba, device);
1855                 sas_put_local_phy(phy);
1856         } else {
1857                 struct hisi_sas_tmf_task tmf_task = { .tmf =  TMF_LU_RESET };
1858
1859                 rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
1860                 if (rc == TMF_RESP_FUNC_COMPLETE)
1861                         hisi_sas_release_task(hisi_hba, device);
1862         }
1863 out:
1864         if (rc != TMF_RESP_FUNC_COMPLETE)
1865                 dev_err(dev, "lu_reset: for device[%d]:rc= %d\n",
1866                              sas_dev->device_id, rc);
1867         return rc;
1868 }
1869
1870 static int hisi_sas_clear_nexus_ha(struct sas_ha_struct *sas_ha)
1871 {
1872         struct hisi_hba *hisi_hba = sas_ha->lldd_ha;
1873         struct device *dev = hisi_hba->dev;
1874         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
1875         int rc, i;
1876
1877         queue_work(hisi_hba->wq, &r.work);
1878         wait_for_completion(r.completion);
1879         if (!r.done)
1880                 return TMF_RESP_FUNC_FAILED;
1881
1882         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
1883                 struct hisi_sas_device *sas_dev = &hisi_hba->devices[i];
1884                 struct domain_device *device = sas_dev->sas_device;
1885
1886                 if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device ||
1887                     dev_is_expander(device->dev_type))
1888                         continue;
1889
1890                 rc = hisi_sas_debug_I_T_nexus_reset(device);
1891                 if (rc != TMF_RESP_FUNC_COMPLETE)
1892                         dev_info(dev, "clear nexus ha: for device[%d] rc=%d\n",
1893                                  sas_dev->device_id, rc);
1894         }
1895
1896         hisi_sas_release_tasks(hisi_hba);
1897
1898         return TMF_RESP_FUNC_COMPLETE;
1899 }
1900
1901 static int hisi_sas_query_task(struct sas_task *task)
1902 {
1903         struct scsi_lun lun;
1904         struct hisi_sas_tmf_task tmf_task;
1905         int rc = TMF_RESP_FUNC_FAILED;
1906
1907         if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) {
1908                 struct scsi_cmnd *cmnd = task->uldd_task;
1909                 struct domain_device *device = task->dev;
1910                 struct hisi_sas_slot *slot = task->lldd_task;
1911                 u32 tag = slot->idx;
1912
1913                 int_to_scsilun(cmnd->device->lun, &lun);
1914                 tmf_task.tmf = TMF_QUERY_TASK;
1915                 tmf_task.tag_of_task_to_be_managed = tag;
1916
1917                 rc = hisi_sas_debug_issue_ssp_tmf(device,
1918                                                   lun.scsi_lun,
1919                                                   &tmf_task);
1920                 switch (rc) {
1921                 /* The task is still in Lun, release it then */
1922                 case TMF_RESP_FUNC_SUCC:
1923                 /* The task is not in Lun or failed, reset the phy */
1924                 case TMF_RESP_FUNC_FAILED:
1925                 case TMF_RESP_FUNC_COMPLETE:
1926                         break;
1927                 default:
1928                         rc = TMF_RESP_FUNC_FAILED;
1929                         break;
1930                 }
1931         }
1932         return rc;
1933 }
1934
1935 static int
1936 hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
1937                                   struct sas_task *task, int abort_flag,
1938                                   int task_tag, struct hisi_sas_dq *dq)
1939 {
1940         struct domain_device *device = task->dev;
1941         struct hisi_sas_device *sas_dev = device->lldd_dev;
1942         struct device *dev = hisi_hba->dev;
1943         struct hisi_sas_port *port;
1944         struct hisi_sas_slot *slot;
1945         struct asd_sas_port *sas_port = device->port;
1946         struct hisi_sas_cmd_hdr *cmd_hdr_base;
1947         int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
1948         unsigned long flags;
1949         int wr_q_index;
1950
1951         if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
1952                 return -EINVAL;
1953
1954         if (!device->port)
1955                 return -1;
1956
1957         port = to_hisi_sas_port(sas_port);
1958
1959         /* simply get a slot and send abort command */
1960         rc = hisi_sas_slot_index_alloc(hisi_hba, NULL);
1961         if (rc < 0)
1962                 goto err_out;
1963
1964         slot_idx = rc;
1965         slot = &hisi_hba->slot_info[slot_idx];
1966
1967         spin_lock_irqsave(&dq->lock, flags);
1968         wr_q_index = dq->wr_point;
1969         dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
1970         list_add_tail(&slot->delivery, &dq->list);
1971         spin_unlock_irqrestore(&dq->lock, flags);
1972         spin_lock_irqsave(&sas_dev->lock, flags);
1973         list_add_tail(&slot->entry, &sas_dev->list);
1974         spin_unlock_irqrestore(&sas_dev->lock, flags);
1975
1976         dlvry_queue = dq->id;
1977         dlvry_queue_slot = wr_q_index;
1978
1979         slot->device_id = sas_dev->device_id;
1980         slot->n_elem = n_elem;
1981         slot->dlvry_queue = dlvry_queue;
1982         slot->dlvry_queue_slot = dlvry_queue_slot;
1983         cmd_hdr_base = hisi_hba->cmd_hdr[dlvry_queue];
1984         slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
1985         slot->task = task;
1986         slot->port = port;
1987         slot->is_internal = true;
1988         task->lldd_task = slot;
1989
1990         memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
1991         memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
1992         memset(hisi_sas_status_buf_addr_mem(slot), 0,
1993                sizeof(struct hisi_sas_err_record));
1994
1995         hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
1996                                       abort_flag, task_tag);
1997
1998         spin_lock_irqsave(&task->task_state_lock, flags);
1999         task->task_state_flags |= SAS_TASK_AT_INITIATOR;
2000         spin_unlock_irqrestore(&task->task_state_lock, flags);
2001         WRITE_ONCE(slot->ready, 1);
2002         /* send abort command to the chip */
2003         spin_lock_irqsave(&dq->lock, flags);
2004         hisi_hba->hw->start_delivery(dq);
2005         spin_unlock_irqrestore(&dq->lock, flags);
2006
2007         return 0;
2008
2009 err_out:
2010         dev_err(dev, "internal abort task prep: failed[%d]!\n", rc);
2011
2012         return rc;
2013 }
2014
2015 /**
2016  * _hisi_sas_internal_task_abort -- execute an internal
2017  * abort command for single IO command or a device
2018  * @hisi_hba: host controller struct
2019  * @device: domain device
2020  * @abort_flag: mode of operation, device or single IO
2021  * @tag: tag of IO to be aborted (only relevant to single
2022  *       IO mode)
2023  * @dq: delivery queue for this internal abort command
2024  */
2025 static int
2026 _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
2027                               struct domain_device *device, int abort_flag,
2028                               int tag, struct hisi_sas_dq *dq)
2029 {
2030         struct sas_task *task;
2031         struct hisi_sas_device *sas_dev = device->lldd_dev;
2032         struct device *dev = hisi_hba->dev;
2033         int res;
2034
2035         /*
2036          * The interface is not realized means this HW don't support internal
2037          * abort, or don't need to do internal abort. Then here, we return
2038          * TMF_RESP_FUNC_FAILED and let other steps go on, which depends that
2039          * the internal abort has been executed and returned CQ.
2040          */
2041         if (!hisi_hba->hw->prep_abort)
2042                 return TMF_RESP_FUNC_FAILED;
2043
2044         task = sas_alloc_slow_task(GFP_KERNEL);
2045         if (!task)
2046                 return -ENOMEM;
2047
2048         task->dev = device;
2049         task->task_proto = device->tproto;
2050         task->task_done = hisi_sas_task_done;
2051         task->slow_task->timer.function = hisi_sas_tmf_timedout;
2052         task->slow_task->timer.expires = jiffies + INTERNAL_ABORT_TIMEOUT * HZ;
2053         add_timer(&task->slow_task->timer);
2054
2055         res = hisi_sas_internal_abort_task_exec(hisi_hba, sas_dev->device_id,
2056                                                 task, abort_flag, tag, dq);
2057         if (res) {
2058                 del_timer(&task->slow_task->timer);
2059                 dev_err(dev, "internal task abort: executing internal task failed: %d\n",
2060                         res);
2061                 goto exit;
2062         }
2063         wait_for_completion(&task->slow_task->completion);
2064         res = TMF_RESP_FUNC_FAILED;
2065
2066         /* Internal abort timed out */
2067         if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) {
2068                 if (hisi_sas_debugfs_enable && hisi_hba->debugfs_itct)
2069                         queue_work(hisi_hba->wq, &hisi_hba->debugfs_work);
2070
2071                 if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
2072                         struct hisi_sas_slot *slot = task->lldd_task;
2073
2074                         if (slot) {
2075                                 struct hisi_sas_cq *cq =
2076                                         &hisi_hba->cq[slot->dlvry_queue];
2077                                 /*
2078                                  * flush tasklet to avoid free'ing task
2079                                  * before using task in IO completion
2080                                  */
2081                                 tasklet_kill(&cq->tasklet);
2082                                 slot->task = NULL;
2083                         }
2084                         dev_err(dev, "internal task abort: timeout and not done.\n");
2085
2086                         res = -EIO;
2087                         goto exit;
2088                 } else
2089                         dev_err(dev, "internal task abort: timeout.\n");
2090         }
2091
2092         if (task->task_status.resp == SAS_TASK_COMPLETE &&
2093                 task->task_status.stat == TMF_RESP_FUNC_COMPLETE) {
2094                 res = TMF_RESP_FUNC_COMPLETE;
2095                 goto exit;
2096         }
2097
2098         if (task->task_status.resp == SAS_TASK_COMPLETE &&
2099                 task->task_status.stat == TMF_RESP_FUNC_SUCC) {
2100                 res = TMF_RESP_FUNC_SUCC;
2101                 goto exit;
2102         }
2103
2104 exit:
2105         dev_dbg(dev, "internal task abort: task to dev %016llx task=%pK resp: 0x%x sts 0x%x\n",
2106                 SAS_ADDR(device->sas_addr), task,
2107                 task->task_status.resp, /* 0 is complete, -1 is undelivered */
2108                 task->task_status.stat);
2109         sas_free_task(task);
2110
2111         return res;
2112 }
2113
2114 static int
2115 hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
2116                              struct domain_device *device,
2117                              int abort_flag, int tag)
2118 {
2119         struct hisi_sas_slot *slot;
2120         struct device *dev = hisi_hba->dev;
2121         struct hisi_sas_dq *dq;
2122         int i, rc;
2123
2124         switch (abort_flag) {
2125         case HISI_SAS_INT_ABT_CMD:
2126                 slot = &hisi_hba->slot_info[tag];
2127                 dq = &hisi_hba->dq[slot->dlvry_queue];
2128                 return _hisi_sas_internal_task_abort(hisi_hba, device,
2129                                                      abort_flag, tag, dq);
2130         case HISI_SAS_INT_ABT_DEV:
2131                 for (i = 0; i < hisi_hba->cq_nvecs; i++) {
2132                         struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2133                         const struct cpumask *mask = cq->pci_irq_mask;
2134
2135                         if (mask && !cpumask_intersects(cpu_online_mask, mask))
2136                                 continue;
2137                         dq = &hisi_hba->dq[i];
2138                         rc = _hisi_sas_internal_task_abort(hisi_hba, device,
2139                                                            abort_flag, tag,
2140                                                            dq);
2141                         if (rc)
2142                                 return rc;
2143                 }
2144                 break;
2145         default:
2146                 dev_err(dev, "Unrecognised internal abort flag (%d)\n",
2147                         abort_flag);
2148                 return -EINVAL;
2149         }
2150
2151         return 0;
2152 }
2153
2154 static void hisi_sas_port_formed(struct asd_sas_phy *sas_phy)
2155 {
2156         hisi_sas_port_notify_formed(sas_phy);
2157 }
2158
2159 static int hisi_sas_write_gpio(struct sas_ha_struct *sha, u8 reg_type,
2160                         u8 reg_index, u8 reg_count, u8 *write_data)
2161 {
2162         struct hisi_hba *hisi_hba = sha->lldd_ha;
2163
2164         if (!hisi_hba->hw->write_gpio)
2165                 return -EOPNOTSUPP;
2166
2167         return hisi_hba->hw->write_gpio(hisi_hba, reg_type,
2168                                 reg_index, reg_count, write_data);
2169 }
2170
2171 static void hisi_sas_phy_disconnected(struct hisi_sas_phy *phy)
2172 {
2173         struct asd_sas_phy *sas_phy = &phy->sas_phy;
2174         struct sas_phy *sphy = sas_phy->phy;
2175         unsigned long flags;
2176
2177         phy->phy_attached = 0;
2178         phy->phy_type = 0;
2179         phy->port = NULL;
2180
2181         spin_lock_irqsave(&phy->lock, flags);
2182         if (phy->enable)
2183                 sphy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
2184         else
2185                 sphy->negotiated_linkrate = SAS_PHY_DISABLED;
2186         spin_unlock_irqrestore(&phy->lock, flags);
2187 }
2188
2189 void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
2190 {
2191         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
2192         struct asd_sas_phy *sas_phy = &phy->sas_phy;
2193         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
2194         struct device *dev = hisi_hba->dev;
2195
2196         if (rdy) {
2197                 /* Phy down but ready */
2198                 hisi_sas_bytes_dmaed(hisi_hba, phy_no);
2199                 hisi_sas_port_notify_formed(sas_phy);
2200         } else {
2201                 struct hisi_sas_port *port  = phy->port;
2202
2203                 if (test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags) ||
2204                     phy->in_reset) {
2205                         dev_info(dev, "ignore flutter phy%d down\n", phy_no);
2206                         return;
2207                 }
2208                 /* Phy down and not ready */
2209                 sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL);
2210                 sas_phy_disconnected(sas_phy);
2211
2212                 if (port) {
2213                         if (phy->phy_type & PORT_TYPE_SAS) {
2214                                 int port_id = port->id;
2215
2216                                 if (!hisi_hba->hw->get_wideport_bitmap(hisi_hba,
2217                                                                        port_id))
2218                                         port->port_attached = 0;
2219                         } else if (phy->phy_type & PORT_TYPE_SATA)
2220                                 port->port_attached = 0;
2221                 }
2222                 hisi_sas_phy_disconnected(phy);
2223         }
2224 }
2225 EXPORT_SYMBOL_GPL(hisi_sas_phy_down);
2226
2227 void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba)
2228 {
2229         int i;
2230
2231         for (i = 0; i < hisi_hba->cq_nvecs; i++) {
2232                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2233
2234                 tasklet_kill(&cq->tasklet);
2235         }
2236 }
2237 EXPORT_SYMBOL_GPL(hisi_sas_kill_tasklets);
2238
2239 int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type)
2240 {
2241         struct hisi_hba *hisi_hba = shost_priv(shost);
2242
2243         if (reset_type != SCSI_ADAPTER_RESET)
2244                 return -EOPNOTSUPP;
2245
2246         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2247
2248         return 0;
2249 }
2250 EXPORT_SYMBOL_GPL(hisi_sas_host_reset);
2251
2252 struct scsi_transport_template *hisi_sas_stt;
2253 EXPORT_SYMBOL_GPL(hisi_sas_stt);
2254
2255 static struct sas_domain_function_template hisi_sas_transport_ops = {
2256         .lldd_dev_found         = hisi_sas_dev_found,
2257         .lldd_dev_gone          = hisi_sas_dev_gone,
2258         .lldd_execute_task      = hisi_sas_queue_command,
2259         .lldd_control_phy       = hisi_sas_control_phy,
2260         .lldd_abort_task        = hisi_sas_abort_task,
2261         .lldd_abort_task_set    = hisi_sas_abort_task_set,
2262         .lldd_clear_aca         = hisi_sas_clear_aca,
2263         .lldd_I_T_nexus_reset   = hisi_sas_I_T_nexus_reset,
2264         .lldd_lu_reset          = hisi_sas_lu_reset,
2265         .lldd_query_task        = hisi_sas_query_task,
2266         .lldd_clear_nexus_ha    = hisi_sas_clear_nexus_ha,
2267         .lldd_port_formed       = hisi_sas_port_formed,
2268         .lldd_write_gpio        = hisi_sas_write_gpio,
2269 };
2270
2271 void hisi_sas_init_mem(struct hisi_hba *hisi_hba)
2272 {
2273         int i, s, j, max_command_entries = HISI_SAS_MAX_COMMANDS;
2274         struct hisi_sas_breakpoint *sata_breakpoint = hisi_hba->sata_breakpoint;
2275
2276         for (i = 0; i < hisi_hba->queue_count; i++) {
2277                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2278                 struct hisi_sas_dq *dq = &hisi_hba->dq[i];
2279                 struct hisi_sas_cmd_hdr *cmd_hdr = hisi_hba->cmd_hdr[i];
2280
2281                 s = sizeof(struct hisi_sas_cmd_hdr);
2282                 for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++)
2283                         memset(&cmd_hdr[j], 0, s);
2284
2285                 dq->wr_point = 0;
2286
2287                 s = hisi_hba->hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS;
2288                 memset(hisi_hba->complete_hdr[i], 0, s);
2289                 cq->rd_point = 0;
2290         }
2291
2292         s = sizeof(struct hisi_sas_initial_fis) * hisi_hba->n_phy;
2293         memset(hisi_hba->initial_fis, 0, s);
2294
2295         s = max_command_entries * sizeof(struct hisi_sas_iost);
2296         memset(hisi_hba->iost, 0, s);
2297
2298         s = max_command_entries * sizeof(struct hisi_sas_breakpoint);
2299         memset(hisi_hba->breakpoint, 0, s);
2300
2301         s = sizeof(struct hisi_sas_sata_breakpoint);
2302         for (j = 0; j < HISI_SAS_MAX_ITCT_ENTRIES; j++)
2303                 memset(&sata_breakpoint[j], 0, s);
2304 }
2305 EXPORT_SYMBOL_GPL(hisi_sas_init_mem);
2306
2307 int hisi_sas_alloc(struct hisi_hba *hisi_hba)
2308 {
2309         struct device *dev = hisi_hba->dev;
2310         int i, j, s, max_command_entries = HISI_SAS_MAX_COMMANDS;
2311         int max_command_entries_ru, sz_slot_buf_ru;
2312         int blk_cnt, slots_per_blk;
2313
2314         sema_init(&hisi_hba->sem, 1);
2315         spin_lock_init(&hisi_hba->lock);
2316         for (i = 0; i < hisi_hba->n_phy; i++) {
2317                 hisi_sas_phy_init(hisi_hba, i);
2318                 hisi_hba->port[i].port_attached = 0;
2319                 hisi_hba->port[i].id = -1;
2320         }
2321
2322         for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
2323                 hisi_hba->devices[i].dev_type = SAS_PHY_UNUSED;
2324                 hisi_hba->devices[i].device_id = i;
2325                 hisi_hba->devices[i].dev_status = HISI_SAS_DEV_INIT;
2326         }
2327
2328         for (i = 0; i < hisi_hba->queue_count; i++) {
2329                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2330                 struct hisi_sas_dq *dq = &hisi_hba->dq[i];
2331
2332                 /* Completion queue structure */
2333                 cq->id = i;
2334                 cq->hisi_hba = hisi_hba;
2335
2336                 /* Delivery queue structure */
2337                 spin_lock_init(&dq->lock);
2338                 INIT_LIST_HEAD(&dq->list);
2339                 dq->id = i;
2340                 dq->hisi_hba = hisi_hba;
2341
2342                 /* Delivery queue */
2343                 s = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS;
2344                 hisi_hba->cmd_hdr[i] = dmam_alloc_coherent(dev, s,
2345                                                 &hisi_hba->cmd_hdr_dma[i],
2346                                                 GFP_KERNEL);
2347                 if (!hisi_hba->cmd_hdr[i])
2348                         goto err_out;
2349
2350                 /* Completion queue */
2351                 s = hisi_hba->hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS;
2352                 hisi_hba->complete_hdr[i] = dmam_alloc_coherent(dev, s,
2353                                                 &hisi_hba->complete_hdr_dma[i],
2354                                                 GFP_KERNEL);
2355                 if (!hisi_hba->complete_hdr[i])
2356                         goto err_out;
2357         }
2358
2359         s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
2360         hisi_hba->itct = dmam_alloc_coherent(dev, s, &hisi_hba->itct_dma,
2361                                              GFP_KERNEL);
2362         if (!hisi_hba->itct)
2363                 goto err_out;
2364
2365         hisi_hba->slot_info = devm_kcalloc(dev, max_command_entries,
2366                                            sizeof(struct hisi_sas_slot),
2367                                            GFP_KERNEL);
2368         if (!hisi_hba->slot_info)
2369                 goto err_out;
2370
2371         /* roundup to avoid overly large block size */
2372         max_command_entries_ru = roundup(max_command_entries, 64);
2373         if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK)
2374                 sz_slot_buf_ru = sizeof(struct hisi_sas_slot_dif_buf_table);
2375         else
2376                 sz_slot_buf_ru = sizeof(struct hisi_sas_slot_buf_table);
2377         sz_slot_buf_ru = roundup(sz_slot_buf_ru, 64);
2378         s = max(lcm(max_command_entries_ru, sz_slot_buf_ru), PAGE_SIZE);
2379         blk_cnt = (max_command_entries_ru * sz_slot_buf_ru) / s;
2380         slots_per_blk = s / sz_slot_buf_ru;
2381
2382         for (i = 0; i < blk_cnt; i++) {
2383                 int slot_index = i * slots_per_blk;
2384                 dma_addr_t buf_dma;
2385                 void *buf;
2386
2387                 buf = dmam_alloc_coherent(dev, s, &buf_dma,
2388                                           GFP_KERNEL);
2389                 if (!buf)
2390                         goto err_out;
2391
2392                 for (j = 0; j < slots_per_blk; j++, slot_index++) {
2393                         struct hisi_sas_slot *slot;
2394
2395                         slot = &hisi_hba->slot_info[slot_index];
2396                         slot->buf = buf;
2397                         slot->buf_dma = buf_dma;
2398                         slot->idx = slot_index;
2399
2400                         buf += sz_slot_buf_ru;
2401                         buf_dma += sz_slot_buf_ru;
2402                 }
2403         }
2404
2405         s = max_command_entries * sizeof(struct hisi_sas_iost);
2406         hisi_hba->iost = dmam_alloc_coherent(dev, s, &hisi_hba->iost_dma,
2407                                              GFP_KERNEL);
2408         if (!hisi_hba->iost)
2409                 goto err_out;
2410
2411         s = max_command_entries * sizeof(struct hisi_sas_breakpoint);
2412         hisi_hba->breakpoint = dmam_alloc_coherent(dev, s,
2413                                                    &hisi_hba->breakpoint_dma,
2414                                                    GFP_KERNEL);
2415         if (!hisi_hba->breakpoint)
2416                 goto err_out;
2417
2418         hisi_hba->slot_index_count = max_command_entries;
2419         s = hisi_hba->slot_index_count / BITS_PER_BYTE;
2420         hisi_hba->slot_index_tags = devm_kzalloc(dev, s, GFP_KERNEL);
2421         if (!hisi_hba->slot_index_tags)
2422                 goto err_out;
2423
2424         s = sizeof(struct hisi_sas_initial_fis) * HISI_SAS_MAX_PHYS;
2425         hisi_hba->initial_fis = dmam_alloc_coherent(dev, s,
2426                                                     &hisi_hba->initial_fis_dma,
2427                                                     GFP_KERNEL);
2428         if (!hisi_hba->initial_fis)
2429                 goto err_out;
2430
2431         s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_sata_breakpoint);
2432         hisi_hba->sata_breakpoint = dmam_alloc_coherent(dev, s,
2433                                         &hisi_hba->sata_breakpoint_dma,
2434                                         GFP_KERNEL);
2435         if (!hisi_hba->sata_breakpoint)
2436                 goto err_out;
2437
2438         hisi_sas_slot_index_init(hisi_hba);
2439         hisi_hba->last_slot_index = HISI_SAS_UNRESERVED_IPTT;
2440
2441         hisi_hba->wq = create_singlethread_workqueue(dev_name(dev));
2442         if (!hisi_hba->wq) {
2443                 dev_err(dev, "sas_alloc: failed to create workqueue\n");
2444                 goto err_out;
2445         }
2446
2447         return 0;
2448 err_out:
2449         return -ENOMEM;
2450 }
2451 EXPORT_SYMBOL_GPL(hisi_sas_alloc);
2452
2453 void hisi_sas_free(struct hisi_hba *hisi_hba)
2454 {
2455         int i;
2456
2457         for (i = 0; i < hisi_hba->n_phy; i++) {
2458                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
2459
2460                 del_timer_sync(&phy->timer);
2461         }
2462
2463         if (hisi_hba->wq)
2464                 destroy_workqueue(hisi_hba->wq);
2465 }
2466 EXPORT_SYMBOL_GPL(hisi_sas_free);
2467
2468 void hisi_sas_rst_work_handler(struct work_struct *work)
2469 {
2470         struct hisi_hba *hisi_hba =
2471                 container_of(work, struct hisi_hba, rst_work);
2472
2473         hisi_sas_controller_reset(hisi_hba);
2474 }
2475 EXPORT_SYMBOL_GPL(hisi_sas_rst_work_handler);
2476
2477 void hisi_sas_sync_rst_work_handler(struct work_struct *work)
2478 {
2479         struct hisi_sas_rst *rst =
2480                 container_of(work, struct hisi_sas_rst, work);
2481
2482         if (!hisi_sas_controller_reset(rst->hisi_hba))
2483                 rst->done = true;
2484         complete(rst->completion);
2485 }
2486 EXPORT_SYMBOL_GPL(hisi_sas_sync_rst_work_handler);
2487
2488 int hisi_sas_get_fw_info(struct hisi_hba *hisi_hba)
2489 {
2490         struct device *dev = hisi_hba->dev;
2491         struct platform_device *pdev = hisi_hba->platform_dev;
2492         struct device_node *np = pdev ? pdev->dev.of_node : NULL;
2493         struct clk *refclk;
2494
2495         if (device_property_read_u8_array(dev, "sas-addr", hisi_hba->sas_addr,
2496                                           SAS_ADDR_SIZE)) {
2497                 dev_err(dev, "could not get property sas-addr\n");
2498                 return -ENOENT;
2499         }
2500
2501         if (np) {
2502                 /*
2503                  * These properties are only required for platform device-based
2504                  * controller with DT firmware.
2505                  */
2506                 hisi_hba->ctrl = syscon_regmap_lookup_by_phandle(np,
2507                                         "hisilicon,sas-syscon");
2508                 if (IS_ERR(hisi_hba->ctrl)) {
2509                         dev_err(dev, "could not get syscon\n");
2510                         return -ENOENT;
2511                 }
2512
2513                 if (device_property_read_u32(dev, "ctrl-reset-reg",
2514                                              &hisi_hba->ctrl_reset_reg)) {
2515                         dev_err(dev, "could not get property ctrl-reset-reg\n");
2516                         return -ENOENT;
2517                 }
2518
2519                 if (device_property_read_u32(dev, "ctrl-reset-sts-reg",
2520                                              &hisi_hba->ctrl_reset_sts_reg)) {
2521                         dev_err(dev, "could not get property ctrl-reset-sts-reg\n");
2522                         return -ENOENT;
2523                 }
2524
2525                 if (device_property_read_u32(dev, "ctrl-clock-ena-reg",
2526                                              &hisi_hba->ctrl_clock_ena_reg)) {
2527                         dev_err(dev, "could not get property ctrl-clock-ena-reg\n");
2528                         return -ENOENT;
2529                 }
2530         }
2531
2532         refclk = devm_clk_get(dev, NULL);
2533         if (IS_ERR(refclk))
2534                 dev_dbg(dev, "no ref clk property\n");
2535         else
2536                 hisi_hba->refclk_frequency_mhz = clk_get_rate(refclk) / 1000000;
2537
2538         if (device_property_read_u32(dev, "phy-count", &hisi_hba->n_phy)) {
2539                 dev_err(dev, "could not get property phy-count\n");
2540                 return -ENOENT;
2541         }
2542
2543         if (device_property_read_u32(dev, "queue-count",
2544                                      &hisi_hba->queue_count)) {
2545                 dev_err(dev, "could not get property queue-count\n");
2546                 return -ENOENT;
2547         }
2548
2549         return 0;
2550 }
2551 EXPORT_SYMBOL_GPL(hisi_sas_get_fw_info);
2552
2553 static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
2554                                               const struct hisi_sas_hw *hw)
2555 {
2556         struct resource *res;
2557         struct Scsi_Host *shost;
2558         struct hisi_hba *hisi_hba;
2559         struct device *dev = &pdev->dev;
2560         int error;
2561
2562         shost = scsi_host_alloc(hw->sht, sizeof(*hisi_hba));
2563         if (!shost) {
2564                 dev_err(dev, "scsi host alloc failed\n");
2565                 return NULL;
2566         }
2567         hisi_hba = shost_priv(shost);
2568
2569         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2570         hisi_hba->hw = hw;
2571         hisi_hba->dev = dev;
2572         hisi_hba->platform_dev = pdev;
2573         hisi_hba->shost = shost;
2574         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2575
2576         timer_setup(&hisi_hba->timer, NULL, 0);
2577
2578         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2579                 goto err_out;
2580
2581         error = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
2582         if (error)
2583                 error = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
2584
2585         if (error) {
2586                 dev_err(dev, "No usable DMA addressing method\n");
2587                 goto err_out;
2588         }
2589
2590         hisi_hba->regs = devm_platform_ioremap_resource(pdev, 0);
2591         if (IS_ERR(hisi_hba->regs))
2592                 goto err_out;
2593
2594         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2595         if (res) {
2596                 hisi_hba->sgpio_regs = devm_ioremap_resource(dev, res);
2597                 if (IS_ERR(hisi_hba->sgpio_regs))
2598                         goto err_out;
2599         }
2600
2601         if (hisi_sas_alloc(hisi_hba)) {
2602                 hisi_sas_free(hisi_hba);
2603                 goto err_out;
2604         }
2605
2606         return shost;
2607 err_out:
2608         scsi_host_put(shost);
2609         dev_err(dev, "shost alloc failed\n");
2610         return NULL;
2611 }
2612
2613 int hisi_sas_probe(struct platform_device *pdev,
2614                    const struct hisi_sas_hw *hw)
2615 {
2616         struct Scsi_Host *shost;
2617         struct hisi_hba *hisi_hba;
2618         struct device *dev = &pdev->dev;
2619         struct asd_sas_phy **arr_phy;
2620         struct asd_sas_port **arr_port;
2621         struct sas_ha_struct *sha;
2622         int rc, phy_nr, port_nr, i;
2623
2624         shost = hisi_sas_shost_alloc(pdev, hw);
2625         if (!shost)
2626                 return -ENOMEM;
2627
2628         sha = SHOST_TO_SAS_HA(shost);
2629         hisi_hba = shost_priv(shost);
2630         platform_set_drvdata(pdev, sha);
2631
2632         phy_nr = port_nr = hisi_hba->n_phy;
2633
2634         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2635         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2636         if (!arr_phy || !arr_port) {
2637                 rc = -ENOMEM;
2638                 goto err_out_ha;
2639         }
2640
2641         sha->sas_phy = arr_phy;
2642         sha->sas_port = arr_port;
2643         sha->lldd_ha = hisi_hba;
2644
2645         shost->transportt = hisi_sas_stt;
2646         shost->max_id = HISI_SAS_MAX_DEVICES;
2647         shost->max_lun = ~0;
2648         shost->max_channel = 1;
2649         shost->max_cmd_len = 16;
2650         if (hisi_hba->hw->slot_index_alloc) {
2651                 shost->can_queue = HISI_SAS_MAX_COMMANDS;
2652                 shost->cmd_per_lun = HISI_SAS_MAX_COMMANDS;
2653         } else {
2654                 shost->can_queue = HISI_SAS_UNRESERVED_IPTT;
2655                 shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT;
2656         }
2657
2658         sha->sas_ha_name = DRV_NAME;
2659         sha->dev = hisi_hba->dev;
2660         sha->lldd_module = THIS_MODULE;
2661         sha->sas_addr = &hisi_hba->sas_addr[0];
2662         sha->num_phys = hisi_hba->n_phy;
2663         sha->core.shost = hisi_hba->shost;
2664
2665         for (i = 0; i < hisi_hba->n_phy; i++) {
2666                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2667                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2668         }
2669
2670         rc = scsi_add_host(shost, &pdev->dev);
2671         if (rc)
2672                 goto err_out_ha;
2673
2674         rc = sas_register_ha(sha);
2675         if (rc)
2676                 goto err_out_register_ha;
2677
2678         rc = hisi_hba->hw->hw_init(hisi_hba);
2679         if (rc)
2680                 goto err_out_register_ha;
2681
2682         scsi_scan_host(shost);
2683
2684         return 0;
2685
2686 err_out_register_ha:
2687         scsi_remove_host(shost);
2688 err_out_ha:
2689         hisi_sas_debugfs_exit(hisi_hba);
2690         hisi_sas_free(hisi_hba);
2691         scsi_host_put(shost);
2692         return rc;
2693 }
2694 EXPORT_SYMBOL_GPL(hisi_sas_probe);
2695
2696 struct dentry *hisi_sas_debugfs_dir;
2697
2698 static void hisi_sas_debugfs_snapshot_cq_reg(struct hisi_hba *hisi_hba)
2699 {
2700         int queue_entry_size = hisi_hba->hw->complete_hdr_size;
2701         int i;
2702
2703         for (i = 0; i < hisi_hba->queue_count; i++)
2704                 memcpy(hisi_hba->debugfs_complete_hdr[i],
2705                        hisi_hba->complete_hdr[i],
2706                        HISI_SAS_QUEUE_SLOTS * queue_entry_size);
2707 }
2708
2709 static void hisi_sas_debugfs_snapshot_dq_reg(struct hisi_hba *hisi_hba)
2710 {
2711         int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr);
2712         int i;
2713
2714         for (i = 0; i < hisi_hba->queue_count; i++) {
2715                 struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr;
2716                 int j;
2717
2718                 debugfs_cmd_hdr = hisi_hba->debugfs_cmd_hdr[i];
2719                 cmd_hdr = hisi_hba->cmd_hdr[i];
2720
2721                 for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++)
2722                         memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j],
2723                                queue_entry_size);
2724         }
2725 }
2726
2727 static void hisi_sas_debugfs_snapshot_port_reg(struct hisi_hba *hisi_hba)
2728 {
2729         const struct hisi_sas_debugfs_reg *port =
2730                 hisi_hba->hw->debugfs_reg_port;
2731         int i, phy_cnt;
2732         u32 offset;
2733         u32 *databuf;
2734
2735         for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) {
2736                 databuf = (u32 *)hisi_hba->debugfs_port_reg[phy_cnt];
2737                 for (i = 0; i < port->count; i++, databuf++) {
2738                         offset = port->base_off + 4 * i;
2739                         *databuf = port->read_port_reg(hisi_hba, phy_cnt,
2740                                                        offset);
2741                 }
2742         }
2743 }
2744
2745 static void hisi_sas_debugfs_snapshot_global_reg(struct hisi_hba *hisi_hba)
2746 {
2747         u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_GLOBAL];
2748         const struct hisi_sas_hw *hw = hisi_hba->hw;
2749         const struct hisi_sas_debugfs_reg *global =
2750                         hw->debugfs_reg_array[DEBUGFS_GLOBAL];
2751         int i;
2752
2753         for (i = 0; i < global->count; i++, databuf++)
2754                 *databuf = global->read_global_reg(hisi_hba, 4 * i);
2755 }
2756
2757 static void hisi_sas_debugfs_snapshot_axi_reg(struct hisi_hba *hisi_hba)
2758 {
2759         u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_AXI];
2760         const struct hisi_sas_hw *hw = hisi_hba->hw;
2761         const struct hisi_sas_debugfs_reg *axi =
2762                         hw->debugfs_reg_array[DEBUGFS_AXI];
2763         int i;
2764
2765         for (i = 0; i < axi->count; i++, databuf++)
2766                 *databuf = axi->read_global_reg(hisi_hba,
2767                                                 4 * i + axi->base_off);
2768 }
2769
2770 static void hisi_sas_debugfs_snapshot_ras_reg(struct hisi_hba *hisi_hba)
2771 {
2772         u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_RAS];
2773         const struct hisi_sas_hw *hw = hisi_hba->hw;
2774         const struct hisi_sas_debugfs_reg *ras =
2775                         hw->debugfs_reg_array[DEBUGFS_RAS];
2776         int i;
2777
2778         for (i = 0; i < ras->count; i++, databuf++)
2779                 *databuf = ras->read_global_reg(hisi_hba,
2780                                                 4 * i + ras->base_off);
2781 }
2782
2783 static void hisi_sas_debugfs_snapshot_itct_reg(struct hisi_hba *hisi_hba)
2784 {
2785         void *cachebuf = hisi_hba->debugfs_itct_cache;
2786         void *databuf = hisi_hba->debugfs_itct;
2787         struct hisi_sas_itct *itct;
2788         int i;
2789
2790         hisi_hba->hw->read_iost_itct_cache(hisi_hba, HISI_SAS_ITCT_CACHE,
2791                                            cachebuf);
2792
2793         itct = hisi_hba->itct;
2794
2795         for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) {
2796                 memcpy(databuf, itct, sizeof(struct hisi_sas_itct));
2797                 databuf += sizeof(struct hisi_sas_itct);
2798         }
2799 }
2800
2801 static void hisi_sas_debugfs_snapshot_iost_reg(struct hisi_hba *hisi_hba)
2802 {
2803         int max_command_entries = HISI_SAS_MAX_COMMANDS;
2804         void *cachebuf = hisi_hba->debugfs_iost_cache;
2805         void *databuf = hisi_hba->debugfs_iost;
2806         struct hisi_sas_iost *iost;
2807         int i;
2808
2809         hisi_hba->hw->read_iost_itct_cache(hisi_hba, HISI_SAS_IOST_CACHE,
2810                                            cachebuf);
2811
2812         iost = hisi_hba->iost;
2813
2814         for (i = 0; i < max_command_entries; i++, iost++) {
2815                 memcpy(databuf, iost, sizeof(struct hisi_sas_iost));
2816                 databuf += sizeof(struct hisi_sas_iost);
2817         }
2818 }
2819
2820 static const char *
2821 hisi_sas_debugfs_to_reg_name(int off, int base_off,
2822                              const struct hisi_sas_debugfs_reg_lu *lu)
2823 {
2824         for (; lu->name; lu++) {
2825                 if (off == lu->off - base_off)
2826                         return lu->name;
2827         }
2828
2829         return NULL;
2830 }
2831
2832 static void hisi_sas_debugfs_print_reg(u32 *regs_val, const void *ptr,
2833                                        struct seq_file *s)
2834 {
2835         const struct hisi_sas_debugfs_reg *reg = ptr;
2836         int i;
2837
2838         for (i = 0; i < reg->count; i++) {
2839                 int off = i * 4;
2840                 const char *name;
2841
2842                 name = hisi_sas_debugfs_to_reg_name(off, reg->base_off,
2843                                                     reg->lu);
2844
2845                 if (name)
2846                         seq_printf(s, "0x%08x 0x%08x %s\n", off,
2847                                    regs_val[i], name);
2848                 else
2849                         seq_printf(s, "0x%08x 0x%08x\n", off,
2850                                    regs_val[i]);
2851         }
2852 }
2853
2854 static int hisi_sas_debugfs_global_show(struct seq_file *s, void *p)
2855 {
2856         struct hisi_hba *hisi_hba = s->private;
2857         const struct hisi_sas_hw *hw = hisi_hba->hw;
2858         const void *reg_global = hw->debugfs_reg_array[DEBUGFS_GLOBAL];
2859
2860         hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_GLOBAL],
2861                                    reg_global, s);
2862
2863         return 0;
2864 }
2865
2866 static int hisi_sas_debugfs_global_open(struct inode *inode, struct file *filp)
2867 {
2868         return single_open(filp, hisi_sas_debugfs_global_show,
2869                            inode->i_private);
2870 }
2871
2872 static const struct file_operations hisi_sas_debugfs_global_fops = {
2873         .open = hisi_sas_debugfs_global_open,
2874         .read = seq_read,
2875         .llseek = seq_lseek,
2876         .release = single_release,
2877         .owner = THIS_MODULE,
2878 };
2879
2880 static int hisi_sas_debugfs_axi_show(struct seq_file *s, void *p)
2881 {
2882         struct hisi_hba *hisi_hba = s->private;
2883         const struct hisi_sas_hw *hw = hisi_hba->hw;
2884         const void *reg_axi = hw->debugfs_reg_array[DEBUGFS_AXI];
2885
2886         hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_AXI],
2887                                    reg_axi, s);
2888
2889         return 0;
2890 }
2891
2892 static int hisi_sas_debugfs_axi_open(struct inode *inode, struct file *filp)
2893 {
2894         return single_open(filp, hisi_sas_debugfs_axi_show,
2895                            inode->i_private);
2896 }
2897
2898 static const struct file_operations hisi_sas_debugfs_axi_fops = {
2899         .open = hisi_sas_debugfs_axi_open,
2900         .read = seq_read,
2901         .llseek = seq_lseek,
2902         .release = single_release,
2903         .owner = THIS_MODULE,
2904 };
2905
2906 static int hisi_sas_debugfs_ras_show(struct seq_file *s, void *p)
2907 {
2908         struct hisi_hba *hisi_hba = s->private;
2909         const struct hisi_sas_hw *hw = hisi_hba->hw;
2910         const void *reg_ras = hw->debugfs_reg_array[DEBUGFS_RAS];
2911
2912         hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_RAS],
2913                                    reg_ras, s);
2914
2915         return 0;
2916 }
2917
2918 static int hisi_sas_debugfs_ras_open(struct inode *inode, struct file *filp)
2919 {
2920         return single_open(filp, hisi_sas_debugfs_ras_show,
2921                            inode->i_private);
2922 }
2923
2924 static const struct file_operations hisi_sas_debugfs_ras_fops = {
2925         .open = hisi_sas_debugfs_ras_open,
2926         .read = seq_read,
2927         .llseek = seq_lseek,
2928         .release = single_release,
2929         .owner = THIS_MODULE,
2930 };
2931
2932 static int hisi_sas_debugfs_port_show(struct seq_file *s, void *p)
2933 {
2934         struct hisi_sas_phy *phy = s->private;
2935         struct hisi_hba *hisi_hba = phy->hisi_hba;
2936         const struct hisi_sas_hw *hw = hisi_hba->hw;
2937         const struct hisi_sas_debugfs_reg *reg_port = hw->debugfs_reg_port;
2938         u32 *databuf = hisi_hba->debugfs_port_reg[phy->sas_phy.id];
2939
2940         hisi_sas_debugfs_print_reg(databuf, reg_port, s);
2941
2942         return 0;
2943 }
2944
2945 static int hisi_sas_debugfs_port_open(struct inode *inode, struct file *filp)
2946 {
2947         return single_open(filp, hisi_sas_debugfs_port_show, inode->i_private);
2948 }
2949
2950 static const struct file_operations hisi_sas_debugfs_port_fops = {
2951         .open = hisi_sas_debugfs_port_open,
2952         .read = seq_read,
2953         .llseek = seq_lseek,
2954         .release = single_release,
2955         .owner = THIS_MODULE,
2956 };
2957
2958 static void hisi_sas_show_row_64(struct seq_file *s, int index,
2959                                  int sz, __le64 *ptr)
2960 {
2961         int i;
2962
2963         /* completion header size not fixed per HW version */
2964         seq_printf(s, "index %04d:\n\t", index);
2965         for (i = 1; i <= sz / 8; i++, ptr++) {
2966                 seq_printf(s, " 0x%016llx", le64_to_cpu(*ptr));
2967                 if (!(i % 2))
2968                         seq_puts(s, "\n\t");
2969         }
2970
2971         seq_puts(s, "\n");
2972 }
2973
2974 static void hisi_sas_show_row_32(struct seq_file *s, int index,
2975                                  int sz, __le32 *ptr)
2976 {
2977         int i;
2978
2979         /* completion header size not fixed per HW version */
2980         seq_printf(s, "index %04d:\n\t", index);
2981         for (i = 1; i <= sz / 4; i++, ptr++) {
2982                 seq_printf(s, " 0x%08x", le32_to_cpu(*ptr));
2983                 if (!(i % 4))
2984                         seq_puts(s, "\n\t");
2985         }
2986         seq_puts(s, "\n");
2987 }
2988
2989 static void hisi_sas_cq_show_slot(struct seq_file *s, int slot, void *cq_ptr)
2990 {
2991         struct hisi_sas_cq *cq = cq_ptr;
2992         struct hisi_hba *hisi_hba = cq->hisi_hba;
2993         void *complete_queue = hisi_hba->debugfs_complete_hdr[cq->id];
2994         __le32 *complete_hdr = complete_queue +
2995                         (hisi_hba->hw->complete_hdr_size * slot);
2996
2997         hisi_sas_show_row_32(s, slot,
2998                              hisi_hba->hw->complete_hdr_size,
2999                              complete_hdr);
3000 }
3001
3002 static int hisi_sas_debugfs_cq_show(struct seq_file *s, void *p)
3003 {
3004         struct hisi_sas_cq *cq = s->private;
3005         int slot;
3006
3007         for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) {
3008                 hisi_sas_cq_show_slot(s, slot, cq);
3009         }
3010         return 0;
3011 }
3012
3013 static int hisi_sas_debugfs_cq_open(struct inode *inode, struct file *filp)
3014 {
3015         return single_open(filp, hisi_sas_debugfs_cq_show, inode->i_private);
3016 }
3017
3018 static const struct file_operations hisi_sas_debugfs_cq_fops = {
3019         .open = hisi_sas_debugfs_cq_open,
3020         .read = seq_read,
3021         .llseek = seq_lseek,
3022         .release = single_release,
3023         .owner = THIS_MODULE,
3024 };
3025
3026 static void hisi_sas_dq_show_slot(struct seq_file *s, int slot, void *dq_ptr)
3027 {
3028         struct hisi_sas_dq *dq = dq_ptr;
3029         struct hisi_hba *hisi_hba = dq->hisi_hba;
3030         void *cmd_queue = hisi_hba->debugfs_cmd_hdr[dq->id];
3031         __le32 *cmd_hdr = cmd_queue +
3032                 sizeof(struct hisi_sas_cmd_hdr) * slot;
3033
3034         hisi_sas_show_row_32(s, slot, sizeof(struct hisi_sas_cmd_hdr), cmd_hdr);
3035 }
3036
3037 static int hisi_sas_debugfs_dq_show(struct seq_file *s, void *p)
3038 {
3039         int slot;
3040
3041         for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) {
3042                 hisi_sas_dq_show_slot(s, slot, s->private);
3043         }
3044         return 0;
3045 }
3046
3047 static int hisi_sas_debugfs_dq_open(struct inode *inode, struct file *filp)
3048 {
3049         return single_open(filp, hisi_sas_debugfs_dq_show, inode->i_private);
3050 }
3051
3052 static const struct file_operations hisi_sas_debugfs_dq_fops = {
3053         .open = hisi_sas_debugfs_dq_open,
3054         .read = seq_read,
3055         .llseek = seq_lseek,
3056         .release = single_release,
3057         .owner = THIS_MODULE,
3058 };
3059
3060 static int hisi_sas_debugfs_iost_show(struct seq_file *s, void *p)
3061 {
3062         struct hisi_hba *hisi_hba = s->private;
3063         struct hisi_sas_iost *debugfs_iost = hisi_hba->debugfs_iost;
3064         int i, max_command_entries = HISI_SAS_MAX_COMMANDS;
3065
3066         for (i = 0; i < max_command_entries; i++, debugfs_iost++) {
3067                 __le64 *iost = &debugfs_iost->qw0;
3068
3069                 hisi_sas_show_row_64(s, i, sizeof(*debugfs_iost), iost);
3070         }
3071
3072         return 0;
3073 }
3074
3075 static int hisi_sas_debugfs_iost_open(struct inode *inode, struct file *filp)
3076 {
3077         return single_open(filp, hisi_sas_debugfs_iost_show, inode->i_private);
3078 }
3079
3080 static const struct file_operations hisi_sas_debugfs_iost_fops = {
3081         .open = hisi_sas_debugfs_iost_open,
3082         .read = seq_read,
3083         .llseek = seq_lseek,
3084         .release = single_release,
3085         .owner = THIS_MODULE,
3086 };
3087
3088 static int hisi_sas_debugfs_iost_cache_show(struct seq_file *s, void *p)
3089 {
3090         struct hisi_hba *hisi_hba = s->private;
3091         struct hisi_sas_iost_itct_cache *iost_cache =
3092                 (struct hisi_sas_iost_itct_cache *)hisi_hba->debugfs_iost_cache;
3093         u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3094         int i, tab_idx;
3095         __le64 *iost;
3096
3097         for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, iost_cache++) {
3098                 /*
3099                  * Data struct of IOST cache:
3100                  * Data[1]: BIT0~15: Table index
3101                  *          Bit16:   Valid mask
3102                  * Data[2]~[9]: IOST table
3103                  */
3104                 tab_idx = (iost_cache->data[1] & 0xffff);
3105                 iost = (__le64 *)iost_cache;
3106
3107                 hisi_sas_show_row_64(s, tab_idx, cache_size, iost);
3108         }
3109
3110         return 0;
3111 }
3112
3113 static int hisi_sas_debugfs_iost_cache_open(struct inode *inode,
3114                                             struct file *filp)
3115 {
3116         return single_open(filp, hisi_sas_debugfs_iost_cache_show,
3117                            inode->i_private);
3118 }
3119
3120 static const struct file_operations hisi_sas_debugfs_iost_cache_fops = {
3121         .open = hisi_sas_debugfs_iost_cache_open,
3122         .read = seq_read,
3123         .llseek = seq_lseek,
3124         .release = single_release,
3125         .owner = THIS_MODULE,
3126 };
3127
3128 static int hisi_sas_debugfs_itct_show(struct seq_file *s, void *p)
3129 {
3130         int i;
3131         struct hisi_hba *hisi_hba = s->private;
3132         struct hisi_sas_itct *debugfs_itct = hisi_hba->debugfs_itct;
3133
3134         for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, debugfs_itct++) {
3135                 __le64 *itct = &debugfs_itct->qw0;
3136
3137                 hisi_sas_show_row_64(s, i, sizeof(*debugfs_itct), itct);
3138         }
3139
3140         return 0;
3141 }
3142
3143 static int hisi_sas_debugfs_itct_open(struct inode *inode, struct file *filp)
3144 {
3145         return single_open(filp, hisi_sas_debugfs_itct_show, inode->i_private);
3146 }
3147
3148 static const struct file_operations hisi_sas_debugfs_itct_fops = {
3149         .open = hisi_sas_debugfs_itct_open,
3150         .read = seq_read,
3151         .llseek = seq_lseek,
3152         .release = single_release,
3153         .owner = THIS_MODULE,
3154 };
3155
3156 static int hisi_sas_debugfs_itct_cache_show(struct seq_file *s, void *p)
3157 {
3158         struct hisi_hba *hisi_hba = s->private;
3159         struct hisi_sas_iost_itct_cache *itct_cache =
3160                 (struct hisi_sas_iost_itct_cache *)hisi_hba->debugfs_itct_cache;
3161         u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3162         int i, tab_idx;
3163         __le64 *itct;
3164
3165         for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, itct_cache++) {
3166                 /*
3167                  * Data struct of ITCT cache:
3168                  * Data[1]: BIT0~15: Table index
3169                  *          Bit16:   Valid mask
3170                  * Data[2]~[9]: ITCT table
3171                  */
3172                 tab_idx = itct_cache->data[1] & 0xffff;
3173                 itct = (__le64 *)itct_cache;
3174
3175                 hisi_sas_show_row_64(s, tab_idx, cache_size, itct);
3176         }
3177
3178         return 0;
3179 }
3180
3181 static int hisi_sas_debugfs_itct_cache_open(struct inode *inode,
3182                                             struct file *filp)
3183 {
3184         return single_open(filp, hisi_sas_debugfs_itct_cache_show,
3185                            inode->i_private);
3186 }
3187
3188 static const struct file_operations hisi_sas_debugfs_itct_cache_fops = {
3189         .open = hisi_sas_debugfs_itct_cache_open,
3190         .read = seq_read,
3191         .llseek = seq_lseek,
3192         .release = single_release,
3193         .owner = THIS_MODULE,
3194 };
3195
3196 static void hisi_sas_debugfs_create_files(struct hisi_hba *hisi_hba)
3197 {
3198         struct dentry *dump_dentry;
3199         struct dentry *dentry;
3200         char name[256];
3201         int p;
3202         int c;
3203         int d;
3204
3205         /* Create dump dir inside device dir */
3206         dump_dentry = debugfs_create_dir("dump", hisi_hba->debugfs_dir);
3207         hisi_hba->debugfs_dump_dentry = dump_dentry;
3208
3209         debugfs_create_file("global", 0400, dump_dentry, hisi_hba,
3210                             &hisi_sas_debugfs_global_fops);
3211
3212         /* Create port dir and files */
3213         dentry = debugfs_create_dir("port", dump_dentry);
3214         for (p = 0; p < hisi_hba->n_phy; p++) {
3215                 snprintf(name, 256, "%d", p);
3216
3217                 debugfs_create_file(name, 0400, dentry, &hisi_hba->phy[p],
3218                                     &hisi_sas_debugfs_port_fops);
3219         }
3220
3221         /* Create CQ dir and files */
3222         dentry = debugfs_create_dir("cq", dump_dentry);
3223         for (c = 0; c < hisi_hba->queue_count; c++) {
3224                 snprintf(name, 256, "%d", c);
3225
3226                 debugfs_create_file(name, 0400, dentry, &hisi_hba->cq[c],
3227                                     &hisi_sas_debugfs_cq_fops);
3228         }
3229
3230         /* Create DQ dir and files */
3231         dentry = debugfs_create_dir("dq", dump_dentry);
3232         for (d = 0; d < hisi_hba->queue_count; d++) {
3233                 snprintf(name, 256, "%d", d);
3234
3235                 debugfs_create_file(name, 0400, dentry, &hisi_hba->dq[d],
3236                                     &hisi_sas_debugfs_dq_fops);
3237         }
3238
3239         debugfs_create_file("iost", 0400, dump_dentry, hisi_hba,
3240                             &hisi_sas_debugfs_iost_fops);
3241
3242         debugfs_create_file("iost_cache", 0400, dump_dentry, hisi_hba,
3243                             &hisi_sas_debugfs_iost_cache_fops);
3244
3245         debugfs_create_file("itct", 0400, dump_dentry, hisi_hba,
3246                             &hisi_sas_debugfs_itct_fops);
3247
3248         debugfs_create_file("itct_cache", 0400, dump_dentry, hisi_hba,
3249                             &hisi_sas_debugfs_itct_cache_fops);
3250
3251         debugfs_create_file("axi", 0400, dump_dentry, hisi_hba,
3252                             &hisi_sas_debugfs_axi_fops);
3253
3254         debugfs_create_file("ras", 0400, dump_dentry, hisi_hba,
3255                             &hisi_sas_debugfs_ras_fops);
3256
3257         return;
3258 }
3259
3260 static void hisi_sas_debugfs_snapshot_regs(struct hisi_hba *hisi_hba)
3261 {
3262         hisi_hba->hw->snapshot_prepare(hisi_hba);
3263
3264         hisi_sas_debugfs_snapshot_global_reg(hisi_hba);
3265         hisi_sas_debugfs_snapshot_port_reg(hisi_hba);
3266         hisi_sas_debugfs_snapshot_axi_reg(hisi_hba);
3267         hisi_sas_debugfs_snapshot_ras_reg(hisi_hba);
3268         hisi_sas_debugfs_snapshot_cq_reg(hisi_hba);
3269         hisi_sas_debugfs_snapshot_dq_reg(hisi_hba);
3270         hisi_sas_debugfs_snapshot_itct_reg(hisi_hba);
3271         hisi_sas_debugfs_snapshot_iost_reg(hisi_hba);
3272
3273         hisi_sas_debugfs_create_files(hisi_hba);
3274
3275         hisi_hba->hw->snapshot_restore(hisi_hba);
3276 }
3277
3278 static ssize_t hisi_sas_debugfs_trigger_dump_write(struct file *file,
3279                                                    const char __user *user_buf,
3280                                                    size_t count, loff_t *ppos)
3281 {
3282         struct hisi_hba *hisi_hba = file->f_inode->i_private;
3283         char buf[8];
3284
3285         /* A bit racy, but don't care too much since it's only debugfs */
3286         if (hisi_hba->debugfs_snapshot)
3287                 return -EFAULT;
3288
3289         if (count > 8)
3290                 return -EFAULT;
3291
3292         if (copy_from_user(buf, user_buf, count))
3293                 return -EFAULT;
3294
3295         if (buf[0] != '1')
3296                 return -EFAULT;
3297
3298         queue_work(hisi_hba->wq, &hisi_hba->debugfs_work);
3299
3300         return count;
3301 }
3302
3303 static const struct file_operations hisi_sas_debugfs_trigger_dump_fops = {
3304         .write = &hisi_sas_debugfs_trigger_dump_write,
3305         .owner = THIS_MODULE,
3306 };
3307
3308 enum {
3309         HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL = 0,
3310         HISI_SAS_BIST_LOOPBACK_MODE_SERDES,
3311         HISI_SAS_BIST_LOOPBACK_MODE_REMOTE,
3312 };
3313
3314 enum {
3315         HISI_SAS_BIST_CODE_MODE_PRBS7 = 0,
3316         HISI_SAS_BIST_CODE_MODE_PRBS23,
3317         HISI_SAS_BIST_CODE_MODE_PRBS31,
3318         HISI_SAS_BIST_CODE_MODE_JTPAT,
3319         HISI_SAS_BIST_CODE_MODE_CJTPAT,
3320         HISI_SAS_BIST_CODE_MODE_SCRAMBED_0,
3321         HISI_SAS_BIST_CODE_MODE_TRAIN,
3322         HISI_SAS_BIST_CODE_MODE_TRAIN_DONE,
3323         HISI_SAS_BIST_CODE_MODE_HFTP,
3324         HISI_SAS_BIST_CODE_MODE_MFTP,
3325         HISI_SAS_BIST_CODE_MODE_LFTP,
3326         HISI_SAS_BIST_CODE_MODE_FIXED_DATA,
3327 };
3328
3329 static const struct {
3330         int             value;
3331         char            *name;
3332 } hisi_sas_debugfs_loop_linkrate[] = {
3333         { SAS_LINK_RATE_1_5_GBPS, "1.5 Gbit" },
3334         { SAS_LINK_RATE_3_0_GBPS, "3.0 Gbit" },
3335         { SAS_LINK_RATE_6_0_GBPS, "6.0 Gbit" },
3336         { SAS_LINK_RATE_12_0_GBPS, "12.0 Gbit" },
3337 };
3338
3339 static int hisi_sas_debugfs_bist_linkrate_show(struct seq_file *s, void *p)
3340 {
3341         struct hisi_hba *hisi_hba = s->private;
3342         int i;
3343
3344         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_linkrate); i++) {
3345                 int match = (hisi_hba->debugfs_bist_linkrate ==
3346                              hisi_sas_debugfs_loop_linkrate[i].value);
3347
3348                 seq_printf(s, "%s%s%s ", match ? "[" : "",
3349                            hisi_sas_debugfs_loop_linkrate[i].name,
3350                            match ? "]" : "");
3351         }
3352         seq_puts(s, "\n");
3353
3354         return 0;
3355 }
3356
3357 static ssize_t hisi_sas_debugfs_bist_linkrate_write(struct file *filp,
3358                                                     const char __user *buf,
3359                                                     size_t count, loff_t *ppos)
3360 {
3361         struct seq_file *m = filp->private_data;
3362         struct hisi_hba *hisi_hba = m->private;
3363         char kbuf[16] = {}, *pkbuf;
3364         bool found = false;
3365         int i;
3366
3367         if (hisi_hba->debugfs_bist_enable)
3368                 return -EPERM;
3369
3370         if (count >= sizeof(kbuf))
3371                 return -EOVERFLOW;
3372
3373         if (copy_from_user(kbuf, buf, count))
3374                 return -EINVAL;
3375
3376         pkbuf = strstrip(kbuf);
3377
3378         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_linkrate); i++) {
3379                 if (!strncmp(hisi_sas_debugfs_loop_linkrate[i].name,
3380                              pkbuf, 16)) {
3381                         hisi_hba->debugfs_bist_linkrate =
3382                                 hisi_sas_debugfs_loop_linkrate[i].value;
3383                         found = true;
3384                         break;
3385                 }
3386         }
3387
3388         if (!found)
3389                 return -EINVAL;
3390
3391         return count;
3392 }
3393
3394 static int hisi_sas_debugfs_bist_linkrate_open(struct inode *inode,
3395                                                struct file *filp)
3396 {
3397         return single_open(filp, hisi_sas_debugfs_bist_linkrate_show,
3398                            inode->i_private);
3399 }
3400
3401 static const struct file_operations hisi_sas_debugfs_bist_linkrate_ops = {
3402         .open = hisi_sas_debugfs_bist_linkrate_open,
3403         .read = seq_read,
3404         .write = hisi_sas_debugfs_bist_linkrate_write,
3405         .llseek = seq_lseek,
3406         .release = single_release,
3407         .owner = THIS_MODULE,
3408 };
3409
3410 static const struct {
3411         int             value;
3412         char            *name;
3413 } hisi_sas_debugfs_loop_code_mode[] = {
3414         { HISI_SAS_BIST_CODE_MODE_PRBS7, "PRBS7" },
3415         { HISI_SAS_BIST_CODE_MODE_PRBS23, "PRBS23" },
3416         { HISI_SAS_BIST_CODE_MODE_PRBS31, "PRBS31" },
3417         { HISI_SAS_BIST_CODE_MODE_JTPAT, "JTPAT" },
3418         { HISI_SAS_BIST_CODE_MODE_CJTPAT, "CJTPAT" },
3419         { HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, "SCRAMBED_0" },
3420         { HISI_SAS_BIST_CODE_MODE_TRAIN, "TRAIN" },
3421         { HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, "TRAIN_DONE" },
3422         { HISI_SAS_BIST_CODE_MODE_HFTP, "HFTP" },
3423         { HISI_SAS_BIST_CODE_MODE_MFTP, "MFTP" },
3424         { HISI_SAS_BIST_CODE_MODE_LFTP, "LFTP" },
3425         { HISI_SAS_BIST_CODE_MODE_FIXED_DATA, "FIXED_DATA" },
3426 };
3427
3428 static int hisi_sas_debugfs_bist_code_mode_show(struct seq_file *s, void *p)
3429 {
3430         struct hisi_hba *hisi_hba = s->private;
3431         int i;
3432
3433         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_code_mode); i++) {
3434                 int match = (hisi_hba->debugfs_bist_code_mode ==
3435                              hisi_sas_debugfs_loop_code_mode[i].value);
3436
3437                 seq_printf(s, "%s%s%s ", match ? "[" : "",
3438                            hisi_sas_debugfs_loop_code_mode[i].name,
3439                            match ? "]" : "");
3440         }
3441         seq_puts(s, "\n");
3442
3443         return 0;
3444 }
3445
3446 static ssize_t hisi_sas_debugfs_bist_code_mode_write(struct file *filp,
3447                                                      const char __user *buf,
3448                                                      size_t count,
3449                                                      loff_t *ppos)
3450 {
3451         struct seq_file *m = filp->private_data;
3452         struct hisi_hba *hisi_hba = m->private;
3453         char kbuf[16] = {}, *pkbuf;
3454         bool found = false;
3455         int i;
3456
3457         if (hisi_hba->debugfs_bist_enable)
3458                 return -EPERM;
3459
3460         if (count >= sizeof(kbuf))
3461                 return -EINVAL;
3462
3463         if (copy_from_user(kbuf, buf, count))
3464                 return -EOVERFLOW;
3465
3466         pkbuf = strstrip(kbuf);
3467
3468         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_code_mode); i++) {
3469                 if (!strncmp(hisi_sas_debugfs_loop_code_mode[i].name,
3470                              pkbuf, 16)) {
3471                         hisi_hba->debugfs_bist_code_mode =
3472                                 hisi_sas_debugfs_loop_code_mode[i].value;
3473                         found = true;
3474                         break;
3475                 }
3476         }
3477
3478         if (!found)
3479                 return -EINVAL;
3480
3481         return count;
3482 }
3483
3484 static int hisi_sas_debugfs_bist_code_mode_open(struct inode *inode,
3485                                                 struct file *filp)
3486 {
3487         return single_open(filp, hisi_sas_debugfs_bist_code_mode_show,
3488                            inode->i_private);
3489 }
3490
3491 static const struct file_operations hisi_sas_debugfs_bist_code_mode_ops = {
3492         .open = hisi_sas_debugfs_bist_code_mode_open,
3493         .read = seq_read,
3494         .write = hisi_sas_debugfs_bist_code_mode_write,
3495         .llseek = seq_lseek,
3496         .release = single_release,
3497         .owner = THIS_MODULE,
3498 };
3499
3500 static ssize_t hisi_sas_debugfs_bist_phy_write(struct file *filp,
3501                                                const char __user *buf,
3502                                                size_t count, loff_t *ppos)
3503 {
3504         struct seq_file *m = filp->private_data;
3505         struct hisi_hba *hisi_hba = m->private;
3506         unsigned int phy_no;
3507         int val;
3508
3509         if (hisi_hba->debugfs_bist_enable)
3510                 return -EPERM;
3511
3512         val = kstrtouint_from_user(buf, count, 0, &phy_no);
3513         if (val)
3514                 return val;
3515
3516         if (phy_no >= hisi_hba->n_phy)
3517                 return -EINVAL;
3518
3519         hisi_hba->debugfs_bist_phy_no = phy_no;
3520
3521         return count;
3522 }
3523
3524 static int hisi_sas_debugfs_bist_phy_show(struct seq_file *s, void *p)
3525 {
3526         struct hisi_hba *hisi_hba = s->private;
3527
3528         seq_printf(s, "%d\n", hisi_hba->debugfs_bist_phy_no);
3529
3530         return 0;
3531 }
3532
3533 static int hisi_sas_debugfs_bist_phy_open(struct inode *inode,
3534                                           struct file *filp)
3535 {
3536         return single_open(filp, hisi_sas_debugfs_bist_phy_show,
3537                            inode->i_private);
3538 }
3539
3540 static const struct file_operations hisi_sas_debugfs_bist_phy_ops = {
3541         .open = hisi_sas_debugfs_bist_phy_open,
3542         .read = seq_read,
3543         .write = hisi_sas_debugfs_bist_phy_write,
3544         .llseek = seq_lseek,
3545         .release = single_release,
3546         .owner = THIS_MODULE,
3547 };
3548
3549 static const struct {
3550         int             value;
3551         char            *name;
3552 } hisi_sas_debugfs_loop_modes[] = {
3553         { HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL, "digial" },
3554         { HISI_SAS_BIST_LOOPBACK_MODE_SERDES, "serdes" },
3555         { HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, "remote" },
3556 };
3557
3558 static int hisi_sas_debugfs_bist_mode_show(struct seq_file *s, void *p)
3559 {
3560         struct hisi_hba *hisi_hba = s->private;
3561         int i;
3562
3563         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_modes); i++) {
3564                 int match = (hisi_hba->debugfs_bist_mode ==
3565                              hisi_sas_debugfs_loop_modes[i].value);
3566
3567                 seq_printf(s, "%s%s%s ", match ? "[" : "",
3568                            hisi_sas_debugfs_loop_modes[i].name,
3569                            match ? "]" : "");
3570         }
3571         seq_puts(s, "\n");
3572
3573         return 0;
3574 }
3575
3576 static ssize_t hisi_sas_debugfs_bist_mode_write(struct file *filp,
3577                                                 const char __user *buf,
3578                                                 size_t count, loff_t *ppos)
3579 {
3580         struct seq_file *m = filp->private_data;
3581         struct hisi_hba *hisi_hba = m->private;
3582         char kbuf[16] = {}, *pkbuf;
3583         bool found = false;
3584         int i;
3585
3586         if (hisi_hba->debugfs_bist_enable)
3587                 return -EPERM;
3588
3589         if (count >= sizeof(kbuf))
3590                 return -EINVAL;
3591
3592         if (copy_from_user(kbuf, buf, count))
3593                 return -EOVERFLOW;
3594
3595         pkbuf = strstrip(kbuf);
3596
3597         for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_modes); i++) {
3598                 if (!strncmp(hisi_sas_debugfs_loop_modes[i].name, pkbuf, 16)) {
3599                         hisi_hba->debugfs_bist_mode =
3600                                 hisi_sas_debugfs_loop_modes[i].value;
3601                         found = true;
3602                         break;
3603                 }
3604         }
3605
3606         if (!found)
3607                 return -EINVAL;
3608
3609         return count;
3610 }
3611
3612 static int hisi_sas_debugfs_bist_mode_open(struct inode *inode,
3613                                            struct file *filp)
3614 {
3615         return single_open(filp, hisi_sas_debugfs_bist_mode_show,
3616                            inode->i_private);
3617 }
3618
3619 static const struct file_operations hisi_sas_debugfs_bist_mode_ops = {
3620         .open = hisi_sas_debugfs_bist_mode_open,
3621         .read = seq_read,
3622         .write = hisi_sas_debugfs_bist_mode_write,
3623         .llseek = seq_lseek,
3624         .release = single_release,
3625         .owner = THIS_MODULE,
3626 };
3627
3628 static ssize_t hisi_sas_debugfs_bist_enable_write(struct file *filp,
3629                                                   const char __user *buf,
3630                                                   size_t count, loff_t *ppos)
3631 {
3632         struct seq_file *m = filp->private_data;
3633         struct hisi_hba *hisi_hba = m->private;
3634         unsigned int enable;
3635         int val;
3636
3637         val = kstrtouint_from_user(buf, count, 0, &enable);
3638         if (val)
3639                 return val;
3640
3641         if (enable > 1)
3642                 return -EINVAL;
3643
3644         if (enable == hisi_hba->debugfs_bist_enable)
3645                 return count;
3646
3647         if (!hisi_hba->hw->set_bist)
3648                 return -EPERM;
3649
3650         val = hisi_hba->hw->set_bist(hisi_hba, enable);
3651         if (val < 0)
3652                 return val;
3653
3654         hisi_hba->debugfs_bist_enable = enable;
3655
3656         return count;
3657 }
3658
3659 static int hisi_sas_debugfs_bist_enable_show(struct seq_file *s, void *p)
3660 {
3661         struct hisi_hba *hisi_hba = s->private;
3662
3663         seq_printf(s, "%d\n", hisi_hba->debugfs_bist_enable);
3664
3665         return 0;
3666 }
3667
3668 static int hisi_sas_debugfs_bist_enable_open(struct inode *inode,
3669                                              struct file *filp)
3670 {
3671         return single_open(filp, hisi_sas_debugfs_bist_enable_show,
3672                            inode->i_private);
3673 }
3674
3675 static const struct file_operations hisi_sas_debugfs_bist_enable_ops = {
3676         .open = hisi_sas_debugfs_bist_enable_open,
3677         .read = seq_read,
3678         .write = hisi_sas_debugfs_bist_enable_write,
3679         .llseek = seq_lseek,
3680         .release = single_release,
3681         .owner = THIS_MODULE,
3682 };
3683
3684 void hisi_sas_debugfs_work_handler(struct work_struct *work)
3685 {
3686         struct hisi_hba *hisi_hba =
3687                 container_of(work, struct hisi_hba, debugfs_work);
3688
3689         if (hisi_hba->debugfs_snapshot)
3690                 return;
3691         hisi_hba->debugfs_snapshot = true;
3692
3693         hisi_sas_debugfs_snapshot_regs(hisi_hba);
3694 }
3695 EXPORT_SYMBOL_GPL(hisi_sas_debugfs_work_handler);
3696
3697 static void hisi_sas_debugfs_release(struct hisi_hba *hisi_hba)
3698 {
3699         struct device *dev = hisi_hba->dev;
3700         int i;
3701
3702         devm_kfree(dev, hisi_hba->debugfs_iost_cache);
3703         devm_kfree(dev, hisi_hba->debugfs_itct_cache);
3704         devm_kfree(dev, hisi_hba->debugfs_iost);
3705
3706         for (i = 0; i < hisi_hba->queue_count; i++)
3707                 devm_kfree(dev, hisi_hba->debugfs_cmd_hdr[i]);
3708
3709         for (i = 0; i < hisi_hba->queue_count; i++)
3710                 devm_kfree(dev, hisi_hba->debugfs_complete_hdr[i]);
3711
3712         for (i = 0; i < DEBUGFS_REGS_NUM; i++)
3713                 devm_kfree(dev, hisi_hba->debugfs_regs[i]);
3714
3715         for (i = 0; i < hisi_hba->n_phy; i++)
3716                 devm_kfree(dev, hisi_hba->debugfs_port_reg[i]);
3717 }
3718
3719 static int hisi_sas_debugfs_alloc(struct hisi_hba *hisi_hba)
3720 {
3721         const struct hisi_sas_hw *hw = hisi_hba->hw;
3722         struct device *dev = hisi_hba->dev;
3723         int p, c, d;
3724         size_t sz;
3725
3726         sz = hw->debugfs_reg_array[DEBUGFS_GLOBAL]->count * 4;
3727         hisi_hba->debugfs_regs[DEBUGFS_GLOBAL] =
3728                                 devm_kmalloc(dev, sz, GFP_KERNEL);
3729
3730         if (!hisi_hba->debugfs_regs[DEBUGFS_GLOBAL])
3731                 goto fail;
3732
3733         sz = hw->debugfs_reg_port->count * 4;
3734         for (p = 0; p < hisi_hba->n_phy; p++) {
3735                 hisi_hba->debugfs_port_reg[p] =
3736                         devm_kmalloc(dev, sz, GFP_KERNEL);
3737
3738                 if (!hisi_hba->debugfs_port_reg[p])
3739                         goto fail;
3740         }
3741
3742         sz = hw->debugfs_reg_array[DEBUGFS_AXI]->count * 4;
3743         hisi_hba->debugfs_regs[DEBUGFS_AXI] =
3744                 devm_kmalloc(dev, sz, GFP_KERNEL);
3745
3746         if (!hisi_hba->debugfs_regs[DEBUGFS_AXI])
3747                 goto fail;
3748
3749         sz = hw->debugfs_reg_array[DEBUGFS_RAS]->count * 4;
3750         hisi_hba->debugfs_regs[DEBUGFS_RAS] =
3751                 devm_kmalloc(dev, sz, GFP_KERNEL);
3752
3753         if (!hisi_hba->debugfs_regs[DEBUGFS_RAS])
3754                 goto fail;
3755
3756         sz = hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS;
3757         for (c = 0; c < hisi_hba->queue_count; c++) {
3758                 hisi_hba->debugfs_complete_hdr[c] =
3759                         devm_kmalloc(dev, sz, GFP_KERNEL);
3760
3761                 if (!hisi_hba->debugfs_complete_hdr[c])
3762                         goto fail;
3763         }
3764
3765         sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS;
3766         for (d = 0; d < hisi_hba->queue_count; d++) {
3767                 hisi_hba->debugfs_cmd_hdr[d] =
3768                         devm_kmalloc(dev, sz, GFP_KERNEL);
3769
3770                 if (!hisi_hba->debugfs_cmd_hdr[d])
3771                         goto fail;
3772         }
3773
3774         sz = HISI_SAS_MAX_COMMANDS * sizeof(struct hisi_sas_iost);
3775
3776         hisi_hba->debugfs_iost = devm_kmalloc(dev, sz, GFP_KERNEL);
3777         if (!hisi_hba->debugfs_iost)
3778                 goto fail;
3779
3780         sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
3781              sizeof(struct hisi_sas_iost_itct_cache);
3782
3783         hisi_hba->debugfs_iost_cache = devm_kmalloc(dev, sz, GFP_KERNEL);
3784         if (!hisi_hba->debugfs_iost_cache)
3785                 goto fail;
3786
3787         sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
3788              sizeof(struct hisi_sas_iost_itct_cache);
3789
3790         hisi_hba->debugfs_itct_cache = devm_kmalloc(dev, sz, GFP_KERNEL);
3791         if (!hisi_hba->debugfs_itct_cache)
3792                 goto fail;
3793
3794         /* New memory allocation must be locate before itct */
3795         sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
3796
3797         hisi_hba->debugfs_itct = devm_kmalloc(dev, sz, GFP_KERNEL);
3798         if (!hisi_hba->debugfs_itct)
3799                 goto fail;
3800
3801         return 0;
3802 fail:
3803         hisi_sas_debugfs_release(hisi_hba);
3804         return -ENOMEM;
3805 }
3806
3807 static void hisi_sas_debugfs_bist_init(struct hisi_hba *hisi_hba)
3808 {
3809         hisi_hba->debugfs_bist_dentry =
3810                         debugfs_create_dir("bist", hisi_hba->debugfs_dir);
3811         debugfs_create_file("link_rate", 0600,
3812                             hisi_hba->debugfs_bist_dentry, hisi_hba,
3813                             &hisi_sas_debugfs_bist_linkrate_ops);
3814
3815         debugfs_create_file("code_mode", 0600,
3816                             hisi_hba->debugfs_bist_dentry, hisi_hba,
3817                             &hisi_sas_debugfs_bist_code_mode_ops);
3818
3819         debugfs_create_file("phy_id", 0600, hisi_hba->debugfs_bist_dentry,
3820                             hisi_hba, &hisi_sas_debugfs_bist_phy_ops);
3821
3822         debugfs_create_u32("cnt", 0600, hisi_hba->debugfs_bist_dentry,
3823                            &hisi_hba->debugfs_bist_cnt);
3824
3825         debugfs_create_file("loopback_mode", 0600,
3826                             hisi_hba->debugfs_bist_dentry,
3827                             hisi_hba, &hisi_sas_debugfs_bist_mode_ops);
3828
3829         debugfs_create_file("enable", 0600, hisi_hba->debugfs_bist_dentry,
3830                             hisi_hba, &hisi_sas_debugfs_bist_enable_ops);
3831
3832         hisi_hba->debugfs_bist_linkrate = SAS_LINK_RATE_1_5_GBPS;
3833 }
3834
3835 void hisi_sas_debugfs_init(struct hisi_hba *hisi_hba)
3836 {
3837         struct device *dev = hisi_hba->dev;
3838
3839         hisi_hba->debugfs_dir = debugfs_create_dir(dev_name(dev),
3840                                                    hisi_sas_debugfs_dir);
3841         debugfs_create_file("trigger_dump", 0600,
3842                             hisi_hba->debugfs_dir,
3843                             hisi_hba,
3844                             &hisi_sas_debugfs_trigger_dump_fops);
3845
3846         /* create bist structures */
3847         hisi_sas_debugfs_bist_init(hisi_hba);
3848
3849         if (hisi_sas_debugfs_alloc(hisi_hba)) {
3850                 debugfs_remove_recursive(hisi_hba->debugfs_dir);
3851                 dev_dbg(dev, "failed to init debugfs!\n");
3852         }
3853 }
3854 EXPORT_SYMBOL_GPL(hisi_sas_debugfs_init);
3855
3856 void hisi_sas_debugfs_exit(struct hisi_hba *hisi_hba)
3857 {
3858         debugfs_remove_recursive(hisi_hba->debugfs_dir);
3859 }
3860 EXPORT_SYMBOL_GPL(hisi_sas_debugfs_exit);
3861
3862 int hisi_sas_remove(struct platform_device *pdev)
3863 {
3864         struct sas_ha_struct *sha = platform_get_drvdata(pdev);
3865         struct hisi_hba *hisi_hba = sha->lldd_ha;
3866         struct Scsi_Host *shost = sha->core.shost;
3867
3868         if (timer_pending(&hisi_hba->timer))
3869                 del_timer(&hisi_hba->timer);
3870
3871         sas_unregister_ha(sha);
3872         sas_remove_host(sha->core.shost);
3873
3874         hisi_sas_free(hisi_hba);
3875         scsi_host_put(shost);
3876         return 0;
3877 }
3878 EXPORT_SYMBOL_GPL(hisi_sas_remove);
3879
3880 bool hisi_sas_debugfs_enable;
3881 EXPORT_SYMBOL_GPL(hisi_sas_debugfs_enable);
3882 module_param_named(debugfs_enable, hisi_sas_debugfs_enable, bool, 0444);
3883 MODULE_PARM_DESC(hisi_sas_debugfs_enable, "Enable driver debugfs (default disabled)");
3884
3885 static __init int hisi_sas_init(void)
3886 {
3887         hisi_sas_stt = sas_domain_attach_transport(&hisi_sas_transport_ops);
3888         if (!hisi_sas_stt)
3889                 return -ENOMEM;
3890
3891         if (hisi_sas_debugfs_enable)
3892                 hisi_sas_debugfs_dir = debugfs_create_dir("hisi_sas", NULL);
3893
3894         return 0;
3895 }
3896
3897 static __exit void hisi_sas_exit(void)
3898 {
3899         sas_release_transport(hisi_sas_stt);
3900
3901         debugfs_remove(hisi_sas_debugfs_dir);
3902 }
3903
3904 module_init(hisi_sas_init);
3905 module_exit(hisi_sas_exit);
3906
3907 MODULE_LICENSE("GPL");
3908 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
3909 MODULE_DESCRIPTION("HISILICON SAS controller driver");
3910 MODULE_ALIAS("platform:" DRV_NAME);