drivers/scsi/qedf/qedf_io.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  QLogic FCoE Offload Driver
   4  *  Copyright (c) 2016-2018 Cavium Inc.
   5  */
   6 #include <linux/spinlock.h>
   7 #include <linux/vmalloc.h>
   8 #include "qedf.h"
   9 #include <scsi/scsi_tcq.h>
  10
  11 void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
  12         unsigned int timer_msec)
  13 {
  14         queue_delayed_work(qedf->timer_work_queue, &io_req->timeout_work,
  15             msecs_to_jiffies(timer_msec));
  16 }
  17
  18 static void qedf_cmd_timeout(struct work_struct *work)
  19 {
  20
  21         struct qedf_ioreq *io_req =
  22             container_of(work, struct qedf_ioreq, timeout_work.work);
  23         struct qedf_ctx *qedf;
  24         struct qedf_rport *fcport;
  25
  26         if (io_req == NULL) {
  27                 QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
  28                 return;
  29         }
  30
  31         fcport = io_req->fcport;
  32         if (io_req->fcport == NULL) {
  33                 QEDF_INFO(NULL, QEDF_LOG_IO,  "fcport is NULL.\n");
  34                 return;
  35         }
  36
  37         qedf = fcport->qedf;
  38
  39         switch (io_req->cmd_type) {
  40         case QEDF_ABTS:
  41                 if (qedf == NULL) {
  42                         QEDF_INFO(NULL, QEDF_LOG_IO,
  43                                   "qedf is NULL for ABTS xid=0x%x.\n",
  44                                   io_req->xid);
  45                         return;
  46                 }
  47
  48                 QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
  49                     io_req->xid);
  50                 /* Cleanup timed out ABTS */
  51                 qedf_initiate_cleanup(io_req, true);
  52                 complete(&io_req->abts_done);
  53
  54                 /*
  55                  * Need to call kref_put for reference taken when initiate_abts
  56                  * was called since abts_compl won't be called now that we've
  57                  * cleaned up the task.
  58                  */
  59                 kref_put(&io_req->refcount, qedf_release_cmd);
  60
  61                 /* Clear in abort bit now that we're done with the command */
  62                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
  63
  64                 /*
  65                  * Now that the original I/O and the ABTS are complete see
  66                  * if we need to reconnect to the target.
  67                  */
  68                 qedf_restart_rport(fcport);
  69                 break;
  70         case QEDF_ELS:
  71                 if (!qedf) {
  72                         QEDF_INFO(NULL, QEDF_LOG_IO,
  73                                   "qedf is NULL for ELS xid=0x%x.\n",
  74                                   io_req->xid);
  75                         return;
  76                 }
  77                 /* ELS request no longer outstanding since it timed out */
  78                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
  79
  80                 kref_get(&io_req->refcount);
  81                 /*
  82                  * Don't attempt to clean an ELS timeout as any subseqeunt
  83                  * ABTS or cleanup requests just hang.  For now just free
  84                  * the resources of the original I/O and the RRQ
  85                  */
  86                 QEDF_ERR(&(qedf->dbg_ctx), "ELS timeout, xid=0x%x.\n",
  87                           io_req->xid);
  88                 qedf_initiate_cleanup(io_req, true);
  89                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
  90                 /* Call callback function to complete command */
  91                 if (io_req->cb_func && io_req->cb_arg) {
  92                         io_req->cb_func(io_req->cb_arg);
  93                         io_req->cb_arg = NULL;
  94                 }
  95                 kref_put(&io_req->refcount, qedf_release_cmd);
  96                 break;
  97         case QEDF_SEQ_CLEANUP:
  98                 QEDF_ERR(&(qedf->dbg_ctx), "Sequence cleanup timeout, "
  99                     "xid=0x%x.\n", io_req->xid);
 100                 qedf_initiate_cleanup(io_req, true);
 101                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
 102                 qedf_process_seq_cleanup_compl(qedf, NULL, io_req);
 103                 break;
 104         default:
 105                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 106                           "Hit default case, xid=0x%x.\n", io_req->xid);
 107                 break;
 108         }
 109 }
 110
 111 void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
 112 {
 113         struct io_bdt *bdt_info;
 114         struct qedf_ctx *qedf = cmgr->qedf;
 115         size_t bd_tbl_sz;
 116         u16 min_xid = 0;
 117         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 118         int num_ios;
 119         int i;
 120         struct qedf_ioreq *io_req;
 121
 122         num_ios = max_xid - min_xid + 1;
 123
 124         /* Free fcoe_bdt_ctx structures */
 125         if (!cmgr->io_bdt_pool) {
 126                 QEDF_ERR(&qedf->dbg_ctx, "io_bdt_pool is NULL.\n");
 127                 goto free_cmd_pool;
 128         }
 129
 130         bd_tbl_sz = QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge);
 131         for (i = 0; i < num_ios; i++) {
 132                 bdt_info = cmgr->io_bdt_pool[i];
 133                 if (bdt_info->bd_tbl) {
 134                         dma_free_coherent(&qedf->pdev->dev, bd_tbl_sz,
 135                             bdt_info->bd_tbl, bdt_info->bd_tbl_dma);
 136                         bdt_info->bd_tbl = NULL;
 137                 }
 138         }
 139
 140         /* Destroy io_bdt pool */
 141         for (i = 0; i < num_ios; i++) {
 142                 kfree(cmgr->io_bdt_pool[i]);
 143                 cmgr->io_bdt_pool[i] = NULL;
 144         }
 145
 146         kfree(cmgr->io_bdt_pool);
 147         cmgr->io_bdt_pool = NULL;
 148
 149 free_cmd_pool:
 150
 151         for (i = 0; i < num_ios; i++) {
 152                 io_req = &cmgr->cmds[i];
 153                 kfree(io_req->sgl_task_params);
 154                 kfree(io_req->task_params);
 155                 /* Make sure we free per command sense buffer */
 156                 if (io_req->sense_buffer)
 157                         dma_free_coherent(&qedf->pdev->dev,
 158                             QEDF_SCSI_SENSE_BUFFERSIZE, io_req->sense_buffer,
 159                             io_req->sense_buffer_dma);
 160                 cancel_delayed_work_sync(&io_req->rrq_work);
 161         }
 162
 163         /* Free command manager itself */
 164         vfree(cmgr);
 165 }
 166
 167 static void qedf_handle_rrq(struct work_struct *work)
 168 {
 169         struct qedf_ioreq *io_req =
 170             container_of(work, struct qedf_ioreq, rrq_work.work);
 171
 172         atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_ACTIVE);
 173         qedf_send_rrq(io_req);
 174
 175 }
 176
 177 struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
 178 {
 179         struct qedf_cmd_mgr *cmgr;
 180         struct io_bdt *bdt_info;
 181         struct qedf_ioreq *io_req;
 182         u16 xid;
 183         int i;
 184         int num_ios;
 185         u16 min_xid = 0;
 186         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 187
 188         /* Make sure num_queues is already set before calling this function */
 189         if (!qedf->num_queues) {
 190                 QEDF_ERR(&(qedf->dbg_ctx), "num_queues is not set.\n");
 191                 return NULL;
 192         }
 193
 194         if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN) {
 195                 QEDF_WARN(&(qedf->dbg_ctx), "Invalid min_xid 0x%x and "
 196                            "max_xid 0x%x.\n", min_xid, max_xid);
 197                 return NULL;
 198         }
 199
 200         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "min xid 0x%x, max xid "
 201                    "0x%x.\n", min_xid, max_xid);
 202
 203         num_ios = max_xid - min_xid + 1;
 204
 205         cmgr = vzalloc(sizeof(struct qedf_cmd_mgr));
 206         if (!cmgr) {
 207                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc cmd mgr.\n");
 208                 return NULL;
 209         }
 210
 211         cmgr->qedf = qedf;
 212         spin_lock_init(&cmgr->lock);
 213
 214         /*
 215          * Initialize I/O request fields.
 216          */
 217         xid = 0;
 218
 219         for (i = 0; i < num_ios; i++) {
 220                 io_req = &cmgr->cmds[i];
 221                 INIT_DELAYED_WORK(&io_req->timeout_work, qedf_cmd_timeout);
 222
 223                 io_req->xid = xid++;
 224
 225                 INIT_DELAYED_WORK(&io_req->rrq_work, qedf_handle_rrq);
 226
 227                 /* Allocate DMA memory to hold sense buffer */
 228                 io_req->sense_buffer = dma_alloc_coherent(&qedf->pdev->dev,
 229                     QEDF_SCSI_SENSE_BUFFERSIZE, &io_req->sense_buffer_dma,
 230                     GFP_KERNEL);
 231                 if (!io_req->sense_buffer) {
 232                         QEDF_ERR(&qedf->dbg_ctx,
 233                                  "Failed to alloc sense buffer.\n");
 234                         goto mem_err;
 235                 }
 236
 237                 /* Allocate task parameters to pass to f/w init funcions */
 238                 io_req->task_params = kzalloc(sizeof(*io_req->task_params),
 239                                               GFP_KERNEL);
 240                 if (!io_req->task_params) {
 241                         QEDF_ERR(&(qedf->dbg_ctx),
 242                                  "Failed to allocate task_params for xid=0x%x\n",
 243                                  i);
 244                         goto mem_err;
 245                 }
 246
 247                 /*
 248                  * Allocate scatter/gather list info to pass to f/w init
 249                  * functions.
 250                  */
 251                 io_req->sgl_task_params = kzalloc(
 252                     sizeof(struct scsi_sgl_task_params), GFP_KERNEL);
 253                 if (!io_req->sgl_task_params) {
 254                         QEDF_ERR(&(qedf->dbg_ctx),
 255                                  "Failed to allocate sgl_task_params for xid=0x%x\n",
 256                                  i);
 257                         goto mem_err;
 258                 }
 259         }
 260
 261         /* Allocate pool of io_bdts - one for each qedf_ioreq */
 262         cmgr->io_bdt_pool = kmalloc_array(num_ios, sizeof(struct io_bdt *),
 263             GFP_KERNEL);
 264
 265         if (!cmgr->io_bdt_pool) {
 266                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc io_bdt_pool.\n");
 267                 goto mem_err;
 268         }
 269
 270         for (i = 0; i < num_ios; i++) {
 271                 cmgr->io_bdt_pool[i] = kmalloc(sizeof(struct io_bdt),
 272                     GFP_KERNEL);
 273                 if (!cmgr->io_bdt_pool[i]) {
 274                         QEDF_WARN(&(qedf->dbg_ctx),
 275                                   "Failed to alloc io_bdt_pool[%d].\n", i);
 276                         goto mem_err;
 277                 }
 278         }
 279
 280         for (i = 0; i < num_ios; i++) {
 281                 bdt_info = cmgr->io_bdt_pool[i];
 282                 bdt_info->bd_tbl = dma_alloc_coherent(&qedf->pdev->dev,
 283                     QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge),
 284                     &bdt_info->bd_tbl_dma, GFP_KERNEL);
 285                 if (!bdt_info->bd_tbl) {
 286                         QEDF_WARN(&(qedf->dbg_ctx),
 287                                   "Failed to alloc bdt_tbl[%d].\n", i);
 288                         goto mem_err;
 289                 }
 290         }
 291         atomic_set(&cmgr->free_list_cnt, num_ios);
 292         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 293             "cmgr->free_list_cnt=%d.\n",
 294             atomic_read(&cmgr->free_list_cnt));
 295
 296         return cmgr;
 297
 298 mem_err:
 299         qedf_cmd_mgr_free(cmgr);
 300         return NULL;
 301 }
 302
 303 struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 304 {
 305         struct qedf_ctx *qedf = fcport->qedf;
 306         struct qedf_cmd_mgr *cmd_mgr = qedf->cmd_mgr;
 307         struct qedf_ioreq *io_req = NULL;
 308         struct io_bdt *bd_tbl;
 309         u16 xid;
 310         uint32_t free_sqes;
 311         int i;
 312         unsigned long flags;
 313
 314         free_sqes = atomic_read(&fcport->free_sqes);
 315
 316         if (!free_sqes) {
 317                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 318                     "Returning NULL, free_sqes=%d.\n ",
 319                     free_sqes);
 320                 goto out_failed;
 321         }
 322
 323         /* Limit the number of outstanding R/W tasks */
 324         if ((atomic_read(&fcport->num_active_ios) >=
 325             NUM_RW_TASKS_PER_CONNECTION)) {
 326                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 327                     "Returning NULL, num_active_ios=%d.\n",
 328                     atomic_read(&fcport->num_active_ios));
 329                 goto out_failed;
 330         }
 331
 332         /* Limit global TIDs certain tasks */
 333         if (atomic_read(&cmd_mgr->free_list_cnt) <= GBL_RSVD_TASKS) {
 334                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 335                     "Returning NULL, free_list_cnt=%d.\n",
 336                     atomic_read(&cmd_mgr->free_list_cnt));
 337                 goto out_failed;
 338         }
 339
 340         spin_lock_irqsave(&cmd_mgr->lock, flags);
 341         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
 342                 io_req = &cmd_mgr->cmds[cmd_mgr->idx];
 343                 cmd_mgr->idx++;
 344                 if (cmd_mgr->idx == FCOE_PARAMS_NUM_TASKS)
 345                         cmd_mgr->idx = 0;
 346
 347                 /* Check to make sure command was previously freed */
 348                 if (!io_req->alloc)
 349                         break;
 350         }
 351
 352         if (i == FCOE_PARAMS_NUM_TASKS) {
 353                 spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 354                 goto out_failed;
 355         }
 356
 357         if (test_bit(QEDF_CMD_DIRTY, &io_req->flags))
 358                 QEDF_ERR(&qedf->dbg_ctx,
 359                          "io_req found to be dirty ox_id = 0x%x.\n",
 360                          io_req->xid);
 361
 362         /* Clear any flags now that we've reallocated the xid */
 363         io_req->flags = 0;
 364         io_req->alloc = 1;
 365         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 366
 367         atomic_inc(&fcport->num_active_ios);
 368         atomic_dec(&fcport->free_sqes);
 369         xid = io_req->xid;
 370         atomic_dec(&cmd_mgr->free_list_cnt);
 371
 372         io_req->cmd_mgr = cmd_mgr;
 373         io_req->fcport = fcport;
 374
 375         /* Clear any stale sc_cmd back pointer */
 376         io_req->sc_cmd = NULL;
 377         io_req->lun = -1;
 378
 379         /* Hold the io_req against deletion */
 380         kref_init(&io_req->refcount);   /* ID: 001 */
 381         atomic_set(&io_req->state, QEDFC_CMD_ST_IO_ACTIVE);
 382
 383         /* Bind io_bdt for this io_req */
 384         /* Have a static link between io_req and io_bdt_pool */
 385         bd_tbl = io_req->bd_tbl = cmd_mgr->io_bdt_pool[xid];
 386         if (bd_tbl == NULL) {
 387                 QEDF_ERR(&(qedf->dbg_ctx), "bd_tbl is NULL, xid=%x.\n", xid);
 388                 kref_put(&io_req->refcount, qedf_release_cmd);
 389                 goto out_failed;
 390         }
 391         bd_tbl->io_req = io_req;
 392         io_req->cmd_type = cmd_type;
 393         io_req->tm_flags = 0;
 394
 395         /* Reset sequence offset data */
 396         io_req->rx_buf_off = 0;
 397         io_req->tx_buf_off = 0;
 398         io_req->rx_id = 0xffff; /* No OX_ID */
 399
 400         return io_req;
 401
 402 out_failed:
 403         /* Record failure for stats and return NULL to caller */
 404         qedf->alloc_failures++;
 405         return NULL;
 406 }
 407
 408 static void qedf_free_mp_resc(struct qedf_ioreq *io_req)
 409 {
 410         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 411         struct qedf_ctx *qedf = io_req->fcport->qedf;
 412         uint64_t sz = sizeof(struct scsi_sge);
 413
 414         /* clear tm flags */
 415         if (mp_req->mp_req_bd) {
 416                 dma_free_coherent(&qedf->pdev->dev, sz,
 417                     mp_req->mp_req_bd, mp_req->mp_req_bd_dma);
 418                 mp_req->mp_req_bd = NULL;
 419         }
 420         if (mp_req->mp_resp_bd) {
 421                 dma_free_coherent(&qedf->pdev->dev, sz,
 422                     mp_req->mp_resp_bd, mp_req->mp_resp_bd_dma);
 423                 mp_req->mp_resp_bd = NULL;
 424         }
 425         if (mp_req->req_buf) {
 426                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 427                     mp_req->req_buf, mp_req->req_buf_dma);
 428                 mp_req->req_buf = NULL;
 429         }
 430         if (mp_req->resp_buf) {
 431                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 432                     mp_req->resp_buf, mp_req->resp_buf_dma);
 433                 mp_req->resp_buf = NULL;
 434         }
 435 }
 436
 437 void qedf_release_cmd(struct kref *ref)
 438 {
 439         struct qedf_ioreq *io_req =
 440             container_of(ref, struct qedf_ioreq, refcount);
 441         struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
 442         struct qedf_rport *fcport = io_req->fcport;
 443         unsigned long flags;
 444
 445         if (io_req->cmd_type == QEDF_SCSI_CMD) {
 446                 QEDF_WARN(&fcport->qedf->dbg_ctx,
 447                           "Cmd released called without scsi_done called, io_req %p xid=0x%x.\n",
 448                           io_req, io_req->xid);
 449                 WARN_ON(io_req->sc_cmd);
 450         }
 451
 452         if (io_req->cmd_type == QEDF_ELS ||
 453             io_req->cmd_type == QEDF_TASK_MGMT_CMD)
 454                 qedf_free_mp_resc(io_req);
 455
 456         atomic_inc(&cmd_mgr->free_list_cnt);
 457         atomic_dec(&fcport->num_active_ios);
 458         atomic_set(&io_req->state, QEDF_CMD_ST_INACTIVE);
 459         if (atomic_read(&fcport->num_active_ios) < 0) {
 460                 QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
 461                 WARN_ON(1);
 462         }
 463
 464         /* Increment task retry identifier now that the request is released */
 465         io_req->task_retry_identifier++;
 466         io_req->fcport = NULL;
 467
 468         clear_bit(QEDF_CMD_DIRTY, &io_req->flags);
 469         io_req->cpu = 0;
 470         spin_lock_irqsave(&cmd_mgr->lock, flags);
 471         io_req->fcport = NULL;
 472         io_req->alloc = 0;
 473         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 474 }
 475
 476 static int qedf_map_sg(struct qedf_ioreq *io_req)
 477 {
 478         struct scsi_cmnd *sc = io_req->sc_cmd;
 479         struct Scsi_Host *host = sc->device->host;
 480         struct fc_lport *lport = shost_priv(host);
 481         struct qedf_ctx *qedf = lport_priv(lport);
 482         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 483         struct scatterlist *sg;
 484         int byte_count = 0;
 485         int sg_count = 0;
 486         int bd_count = 0;
 487         u32 sg_len;
 488         u64 addr;
 489         int i = 0;
 490
 491         sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
 492             scsi_sg_count(sc), sc->sc_data_direction);
 493         sg = scsi_sglist(sc);
 494
 495         io_req->sge_type = QEDF_IOREQ_UNKNOWN_SGE;
 496
 497         if (sg_count <= 8 || io_req->io_req_flags == QEDF_READ)
 498                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 499
 500         scsi_for_each_sg(sc, sg, sg_count, i) {
 501                 sg_len = (u32)sg_dma_len(sg);
 502                 addr = (u64)sg_dma_address(sg);
 503
 504                 /*
 505                  * Intermediate s/g element so check if start address
 506                  * is page aligned.  Only required for writes and only if the
 507                  * number of scatter/gather elements is 8 or more.
 508                  */
 509                 if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE && (i) &&
 510                     (i != (sg_count - 1)) && sg_len < QEDF_PAGE_SIZE)
 511                         io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
 512
 513                 bd[bd_count].sge_addr.lo = cpu_to_le32(U64_LO(addr));
 514                 bd[bd_count].sge_addr.hi  = cpu_to_le32(U64_HI(addr));
 515                 bd[bd_count].sge_len = cpu_to_le32(sg_len);
 516
 517                 bd_count++;
 518                 byte_count += sg_len;
 519         }
 520
 521         /* To catch a case where FAST and SLOW nothing is set, set FAST */
 522         if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE)
 523                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 524
 525         if (byte_count != scsi_bufflen(sc))
 526                 QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
 527                           "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
 528                            scsi_bufflen(sc), io_req->xid);
 529
 530         return bd_count;
 531 }
 532
 533 static int qedf_build_bd_list_from_sg(struct qedf_ioreq *io_req)
 534 {
 535         struct scsi_cmnd *sc = io_req->sc_cmd;
 536         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 537         int bd_count;
 538
 539         if (scsi_sg_count(sc)) {
 540                 bd_count = qedf_map_sg(io_req);
 541                 if (bd_count == 0)
 542                         return -ENOMEM;
 543         } else {
 544                 bd_count = 0;
 545                 bd[0].sge_addr.lo = bd[0].sge_addr.hi = 0;
 546                 bd[0].sge_len = 0;
 547         }
 548         io_req->bd_tbl->bd_valid = bd_count;
 549
 550         return 0;
 551 }
 552
 553 static void qedf_build_fcp_cmnd(struct qedf_ioreq *io_req,
 554                                   struct fcp_cmnd *fcp_cmnd)
 555 {
 556         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 557
 558         /* fcp_cmnd is 32 bytes */
 559         memset(fcp_cmnd, 0, FCP_CMND_LEN);
 560
 561         /* 8 bytes: SCSI LUN info */
 562         int_to_scsilun(sc_cmd->device->lun,
 563                         (struct scsi_lun *)&fcp_cmnd->fc_lun);
 564
 565         /* 4 bytes: flag info */
 566         fcp_cmnd->fc_pri_ta = 0;
 567         fcp_cmnd->fc_tm_flags = io_req->tm_flags;
 568         fcp_cmnd->fc_flags = io_req->io_req_flags;
 569         fcp_cmnd->fc_cmdref = 0;
 570
 571         /* Populate data direction */
 572         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 573                 fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 574         } else {
 575                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
 576                         fcp_cmnd->fc_flags |= FCP_CFL_WRDATA;
 577                 else if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE)
 578                         fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 579         }
 580
 581         fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
 582
 583         /* 16 bytes: CDB information */
 584         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD)
 585                 memcpy(fcp_cmnd->fc_cdb, sc_cmd->cmnd, sc_cmd->cmd_len);
 586
 587         /* 4 bytes: FCP data length */
 588         fcp_cmnd->fc_dl = htonl(io_req->data_xfer_len);
 589 }
 590
 591 static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
 592         struct qedf_ioreq *io_req, struct e4_fcoe_task_context *task_ctx,
 593         struct fcoe_wqe *sqe)
 594 {
 595         enum fcoe_task_type task_type;
 596         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 597         struct io_bdt *bd_tbl = io_req->bd_tbl;
 598         u8 fcp_cmnd[32];
 599         u32 tmp_fcp_cmnd[8];
 600         int bd_count = 0;
 601         struct qedf_ctx *qedf = fcport->qedf;
 602         uint16_t cq_idx = smp_processor_id() % qedf->num_queues;
 603         struct regpair sense_data_buffer_phys_addr;
 604         u32 tx_io_size = 0;
 605         u32 rx_io_size = 0;
 606         int i, cnt;
 607
 608         /* Note init_initiator_rw_fcoe_task memsets the task context */
 609         io_req->task = task_ctx;
 610         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 611         memset(io_req->task_params, 0, sizeof(struct fcoe_task_params));
 612         memset(io_req->sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 613
 614         /* Set task type bassed on DMA directio of command */
 615         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 616                 task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 617         } else {
 618                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 619                         task_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
 620                         tx_io_size = io_req->data_xfer_len;
 621                 } else {
 622                         task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 623                         rx_io_size = io_req->data_xfer_len;
 624                 }
 625         }
 626
 627         /* Setup the fields for fcoe_task_params */
 628         io_req->task_params->context = task_ctx;
 629         io_req->task_params->sqe = sqe;
 630         io_req->task_params->task_type = task_type;
 631         io_req->task_params->tx_io_size = tx_io_size;
 632         io_req->task_params->rx_io_size = rx_io_size;
 633         io_req->task_params->conn_cid = fcport->fw_cid;
 634         io_req->task_params->itid = io_req->xid;
 635         io_req->task_params->cq_rss_number = cq_idx;
 636         io_req->task_params->is_tape_device = fcport->dev_type;
 637
 638         /* Fill in information for scatter/gather list */
 639         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD) {
 640                 bd_count = bd_tbl->bd_valid;
 641                 io_req->sgl_task_params->sgl = bd_tbl->bd_tbl;
 642                 io_req->sgl_task_params->sgl_phys_addr.lo =
 643                         U64_LO(bd_tbl->bd_tbl_dma);
 644                 io_req->sgl_task_params->sgl_phys_addr.hi =
 645                         U64_HI(bd_tbl->bd_tbl_dma);
 646                 io_req->sgl_task_params->num_sges = bd_count;
 647                 io_req->sgl_task_params->total_buffer_size =
 648                     scsi_bufflen(io_req->sc_cmd);
 649                 if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 650                         io_req->sgl_task_params->small_mid_sge = 1;
 651                 else
 652                         io_req->sgl_task_params->small_mid_sge = 0;
 653         }
 654
 655         /* Fill in physical address of sense buffer */
 656         sense_data_buffer_phys_addr.lo = U64_LO(io_req->sense_buffer_dma);
 657         sense_data_buffer_phys_addr.hi = U64_HI(io_req->sense_buffer_dma);
 658
 659         /* fill FCP_CMND IU */
 660         qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)tmp_fcp_cmnd);
 661
 662         /* Swap fcp_cmnd since FC is big endian */
 663         cnt = sizeof(struct fcp_cmnd) / sizeof(u32);
 664         for (i = 0; i < cnt; i++) {
 665                 tmp_fcp_cmnd[i] = cpu_to_be32(tmp_fcp_cmnd[i]);
 666         }
 667         memcpy(fcp_cmnd, tmp_fcp_cmnd, sizeof(struct fcp_cmnd));
 668
 669         init_initiator_rw_fcoe_task(io_req->task_params,
 670                                     io_req->sgl_task_params,
 671                                     sense_data_buffer_phys_addr,
 672                                     io_req->task_retry_identifier, fcp_cmnd);
 673
 674         /* Increment SGL type counters */
 675         if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 676                 qedf->slow_sge_ios++;
 677         else
 678                 qedf->fast_sge_ios++;
 679 }
 680
 681 void qedf_init_mp_task(struct qedf_ioreq *io_req,
 682         struct e4_fcoe_task_context *task_ctx, struct fcoe_wqe *sqe)
 683 {
 684         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 685         struct qedf_rport *fcport = io_req->fcport;
 686         struct qedf_ctx *qedf = io_req->fcport->qedf;
 687         struct fc_frame_header *fc_hdr;
 688         struct fcoe_tx_mid_path_params task_fc_hdr;
 689         struct scsi_sgl_task_params tx_sgl_task_params;
 690         struct scsi_sgl_task_params rx_sgl_task_params;
 691
 692         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 693                   "Initializing MP task for cmd_type=%d\n",
 694                   io_req->cmd_type);
 695
 696         qedf->control_requests++;
 697
 698         memset(&tx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 699         memset(&rx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 700         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 701         memset(&task_fc_hdr, 0, sizeof(struct fcoe_tx_mid_path_params));
 702
 703         /* Setup the task from io_req for easy reference */
 704         io_req->task = task_ctx;
 705
 706         /* Setup the fields for fcoe_task_params */
 707         io_req->task_params->context = task_ctx;
 708         io_req->task_params->sqe = sqe;
 709         io_req->task_params->task_type = FCOE_TASK_TYPE_MIDPATH;
 710         io_req->task_params->tx_io_size = io_req->data_xfer_len;
 711         /* rx_io_size tells the f/w how large a response buffer we have */
 712         io_req->task_params->rx_io_size = PAGE_SIZE;
 713         io_req->task_params->conn_cid = fcport->fw_cid;
 714         io_req->task_params->itid = io_req->xid;
 715         /* Return middle path commands on CQ 0 */
 716         io_req->task_params->cq_rss_number = 0;
 717         io_req->task_params->is_tape_device = fcport->dev_type;
 718
 719         fc_hdr = &(mp_req->req_fc_hdr);
 720         /* Set OX_ID and RX_ID based on driver task id */
 721         fc_hdr->fh_ox_id = io_req->xid;
 722         fc_hdr->fh_rx_id = htons(0xffff);
 723
 724         /* Set up FC header information */
 725         task_fc_hdr.parameter = fc_hdr->fh_parm_offset;
 726         task_fc_hdr.r_ctl = fc_hdr->fh_r_ctl;
 727         task_fc_hdr.type = fc_hdr->fh_type;
 728         task_fc_hdr.cs_ctl = fc_hdr->fh_cs_ctl;
 729         task_fc_hdr.df_ctl = fc_hdr->fh_df_ctl;
 730         task_fc_hdr.rx_id = fc_hdr->fh_rx_id;
 731         task_fc_hdr.ox_id = fc_hdr->fh_ox_id;
 732
 733         /* Set up s/g list parameters for request buffer */
 734         tx_sgl_task_params.sgl = mp_req->mp_req_bd;
 735         tx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_req_bd_dma);
 736         tx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_req_bd_dma);
 737         tx_sgl_task_params.num_sges = 1;
 738         /* Set PAGE_SIZE for now since sg element is that size ??? */
 739         tx_sgl_task_params.total_buffer_size = io_req->data_xfer_len;
 740         tx_sgl_task_params.small_mid_sge = 0;
 741
 742         /* Set up s/g list parameters for request buffer */
 743         rx_sgl_task_params.sgl = mp_req->mp_resp_bd;
 744         rx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_resp_bd_dma);
 745         rx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_resp_bd_dma);
 746         rx_sgl_task_params.num_sges = 1;
 747         /* Set PAGE_SIZE for now since sg element is that size ??? */
 748         rx_sgl_task_params.total_buffer_size = PAGE_SIZE;
 749         rx_sgl_task_params.small_mid_sge = 0;
 750
 751
 752         /*
 753          * Last arg is 0 as previous code did not set that we wanted the
 754          * fc header information.
 755          */
 756         init_initiator_midpath_unsolicited_fcoe_task(io_req->task_params,
 757                                                      &task_fc_hdr,
 758                                                      &tx_sgl_task_params,
 759                                                      &rx_sgl_task_params, 0);
 760 }
 761
 762 /* Presumed that fcport->rport_lock is held */
 763 u16 qedf_get_sqe_idx(struct qedf_rport *fcport)
 764 {
 765         uint16_t total_sqe = (fcport->sq_mem_size)/(sizeof(struct fcoe_wqe));
 766         u16 rval;
 767
 768         rval = fcport->sq_prod_idx;
 769
 770         /* Adjust ring index */
 771         fcport->sq_prod_idx++;
 772         fcport->fw_sq_prod_idx++;
 773         if (fcport->sq_prod_idx == total_sqe)
 774                 fcport->sq_prod_idx = 0;
 775
 776         return rval;
 777 }
 778
 779 void qedf_ring_doorbell(struct qedf_rport *fcport)
 780 {
 781         struct fcoe_db_data dbell = { 0 };
 782
 783         dbell.agg_flags = 0;
 784
 785         dbell.params |= DB_DEST_XCM << FCOE_DB_DATA_DEST_SHIFT;
 786         dbell.params |= DB_AGG_CMD_SET << FCOE_DB_DATA_AGG_CMD_SHIFT;
 787         dbell.params |= DQ_XCM_FCOE_SQ_PROD_CMD <<
 788             FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
 789
 790         dbell.sq_prod = fcport->fw_sq_prod_idx;
 791         /* wmb makes sure that the BDs data is updated before updating the
 792          * producer, otherwise FW may read old data from the BDs.
 793          */
 794         wmb();
 795         barrier();
 796         writel(*(u32 *)&dbell, fcport->p_doorbell);
 797         /*
 798          * Fence required to flush the write combined buffer, since another
 799          * CPU may write to the same doorbell address and data may be lost
 800          * due to relaxed order nature of write combined bar.
 801          */
 802         wmb();
 803 }
 804
 805 static void qedf_trace_io(struct qedf_rport *fcport, struct qedf_ioreq *io_req,
 806                           int8_t direction)
 807 {
 808         struct qedf_ctx *qedf = fcport->qedf;
 809         struct qedf_io_log *io_log;
 810         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 811         unsigned long flags;
 812         uint8_t op;
 813
 814         spin_lock_irqsave(&qedf->io_trace_lock, flags);
 815
 816         io_log = &qedf->io_trace_buf[qedf->io_trace_idx];
 817         io_log->direction = direction;
 818         io_log->task_id = io_req->xid;
 819         io_log->port_id = fcport->rdata->ids.port_id;
 820         io_log->lun = sc_cmd->device->lun;
 821         io_log->op = op = sc_cmd->cmnd[0];
 822         io_log->lba[0] = sc_cmd->cmnd[2];
 823         io_log->lba[1] = sc_cmd->cmnd[3];
 824         io_log->lba[2] = sc_cmd->cmnd[4];
 825         io_log->lba[3] = sc_cmd->cmnd[5];
 826         io_log->bufflen = scsi_bufflen(sc_cmd);
 827         io_log->sg_count = scsi_sg_count(sc_cmd);
 828         io_log->result = sc_cmd->result;
 829         io_log->jiffies = jiffies;
 830         io_log->refcount = kref_read(&io_req->refcount);
 831
 832         if (direction == QEDF_IO_TRACE_REQ) {
 833                 /* For requests we only care abot the submission CPU */
 834                 io_log->req_cpu = io_req->cpu;
 835                 io_log->int_cpu = 0;
 836                 io_log->rsp_cpu = 0;
 837         } else if (direction == QEDF_IO_TRACE_RSP) {
 838                 io_log->req_cpu = io_req->cpu;
 839                 io_log->int_cpu = io_req->int_cpu;
 840                 io_log->rsp_cpu = smp_processor_id();
 841         }
 842
 843         io_log->sge_type = io_req->sge_type;
 844
 845         qedf->io_trace_idx++;
 846         if (qedf->io_trace_idx == QEDF_IO_TRACE_SIZE)
 847                 qedf->io_trace_idx = 0;
 848
 849         spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
 850 }
 851
 852 int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 853 {
 854         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 855         struct Scsi_Host *host = sc_cmd->device->host;
 856         struct fc_lport *lport = shost_priv(host);
 857         struct qedf_ctx *qedf = lport_priv(lport);
 858         struct e4_fcoe_task_context *task_ctx;
 859         u16 xid;
 860         struct fcoe_wqe *sqe;
 861         u16 sqe_idx;
 862
 863         /* Initialize rest of io_req fileds */
 864         io_req->data_xfer_len = scsi_bufflen(sc_cmd);
 865         sc_cmd->SCp.ptr = (char *)io_req;
 866         io_req->sge_type = QEDF_IOREQ_FAST_SGE; /* Assume fast SGL by default */
 867
 868         /* Record which cpu this request is associated with */
 869         io_req->cpu = smp_processor_id();
 870
 871         if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
 872                 io_req->io_req_flags = QEDF_READ;
 873                 qedf->input_requests++;
 874         } else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 875                 io_req->io_req_flags = QEDF_WRITE;
 876                 qedf->output_requests++;
 877         } else {
 878                 io_req->io_req_flags = 0;
 879                 qedf->control_requests++;
 880         }
 881
 882         xid = io_req->xid;
 883
 884         /* Build buffer descriptor list for firmware from sg list */
 885         if (qedf_build_bd_list_from_sg(io_req)) {
 886                 QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
 887                 /* Release cmd will release io_req, but sc_cmd is assigned */
 888                 io_req->sc_cmd = NULL;
 889                 kref_put(&io_req->refcount, qedf_release_cmd);
 890                 return -EAGAIN;
 891         }
 892
 893         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
 894             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 895                 QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
 896                 /* Release cmd will release io_req, but sc_cmd is assigned */
 897                 io_req->sc_cmd = NULL;
 898                 kref_put(&io_req->refcount, qedf_release_cmd);
 899                 return -EINVAL;
 900         }
 901
 902         /* Record LUN number for later use if we neeed them */
 903         io_req->lun = (int)sc_cmd->device->lun;
 904
 905         /* Obtain free SQE */
 906         sqe_idx = qedf_get_sqe_idx(fcport);
 907         sqe = &fcport->sq[sqe_idx];
 908         memset(sqe, 0, sizeof(struct fcoe_wqe));
 909
 910         /* Get the task context */
 911         task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
 912         if (!task_ctx) {
 913                 QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
 914                            xid);
 915                 /* Release cmd will release io_req, but sc_cmd is assigned */
 916                 io_req->sc_cmd = NULL;
 917                 kref_put(&io_req->refcount, qedf_release_cmd);
 918                 return -EINVAL;
 919         }
 920
 921         qedf_init_task(fcport, lport, io_req, task_ctx, sqe);
 922
 923         /* Ring doorbell */
 924         qedf_ring_doorbell(fcport);
 925
 926         /* Set that command is with the firmware now */
 927         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
 928
 929         if (qedf_io_tracing && io_req->sc_cmd)
 930                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
 931
 932         return false;
 933 }
 934
 935 int
 936 qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 937 {
 938         struct fc_lport *lport = shost_priv(host);
 939         struct qedf_ctx *qedf = lport_priv(lport);
 940         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
 941         struct fc_rport_libfc_priv *rp = rport->dd_data;
 942         struct qedf_rport *fcport;
 943         struct qedf_ioreq *io_req;
 944         int rc = 0;
 945         int rval;
 946         unsigned long flags = 0;
 947         int num_sgs = 0;
 948
 949         num_sgs = scsi_sg_count(sc_cmd);
 950         if (scsi_sg_count(sc_cmd) > QEDF_MAX_BDS_PER_CMD) {
 951                 QEDF_ERR(&qedf->dbg_ctx,
 952                          "Number of SG elements %d exceeds what hardware limitation of %d.\n",
 953                          num_sgs, QEDF_MAX_BDS_PER_CMD);
 954                 sc_cmd->result = DID_ERROR;
 955                 sc_cmd->scsi_done(sc_cmd);
 956                 return 0;
 957         }
 958
 959         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
 960             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
 961                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 962                           "Returning DNC as unloading or stop io, flags 0x%lx.\n",
 963                           qedf->flags);
 964                 sc_cmd->result = DID_NO_CONNECT << 16;
 965                 sc_cmd->scsi_done(sc_cmd);
 966                 return 0;
 967         }
 968
 969         if (!qedf->pdev->msix_enabled) {
 970                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 971                     "Completing sc_cmd=%p DID_NO_CONNECT as MSI-X is not enabled.\n",
 972                     sc_cmd);
 973                 sc_cmd->result = DID_NO_CONNECT << 16;
 974                 sc_cmd->scsi_done(sc_cmd);
 975                 return 0;
 976         }
 977
 978         rval = fc_remote_port_chkready(rport);
 979         if (rval) {
 980                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 981                           "fc_remote_port_chkready failed=0x%x for port_id=0x%06x.\n",
 982                           rval, rport->port_id);
 983                 sc_cmd->result = rval;
 984                 sc_cmd->scsi_done(sc_cmd);
 985                 return 0;
 986         }
 987
 988         /* Retry command if we are doing a qed drain operation */
 989         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
 990                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Drain active.\n");
 991                 rc = SCSI_MLQUEUE_HOST_BUSY;
 992                 goto exit_qcmd;
 993         }
 994
 995         if (lport->state != LPORT_ST_READY ||
 996             atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
 997                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Link down.\n");
 998                 rc = SCSI_MLQUEUE_HOST_BUSY;
 999                 goto exit_qcmd;
1000         }
1001
1002         /* rport and tgt are allocated together, so tgt should be non-NULL */
1003         fcport = (struct qedf_rport *)&rp[1];
1004
1005         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
1006             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1007                 /*
1008                  * Session is not offloaded yet. Let SCSI-ml retry
1009                  * the command.
1010                  */
1011                 rc = SCSI_MLQUEUE_TARGET_BUSY;
1012                 goto exit_qcmd;
1013         }
1014
1015         atomic_inc(&fcport->ios_to_queue);
1016
1017         if (fcport->retry_delay_timestamp) {
1018                 /* Take fcport->rport_lock for resetting the delay_timestamp */
1019                 spin_lock_irqsave(&fcport->rport_lock, flags);
1020                 if (time_after(jiffies, fcport->retry_delay_timestamp)) {
1021                         fcport->retry_delay_timestamp = 0;
1022                 } else {
1023                         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1024                         /* If retry_delay timer is active, flow off the ML */
1025                         rc = SCSI_MLQUEUE_TARGET_BUSY;
1026                         atomic_dec(&fcport->ios_to_queue);
1027                         goto exit_qcmd;
1028                 }
1029                 spin_unlock_irqrestore(&fcport->rport_lock, flags);
1030         }
1031
1032         io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
1033         if (!io_req) {
1034                 rc = SCSI_MLQUEUE_HOST_BUSY;
1035                 atomic_dec(&fcport->ios_to_queue);
1036                 goto exit_qcmd;
1037         }
1038
1039         io_req->sc_cmd = sc_cmd;
1040
1041         /* Take fcport->rport_lock for posting to fcport send queue */
1042         spin_lock_irqsave(&fcport->rport_lock, flags);
1043         if (qedf_post_io_req(fcport, io_req)) {
1044                 QEDF_WARN(&(qedf->dbg_ctx), "Unable to post io_req\n");
1045                 /* Return SQE to pool */
1046                 atomic_inc(&fcport->free_sqes);
1047                 rc = SCSI_MLQUEUE_HOST_BUSY;
1048         }
1049         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1050         atomic_dec(&fcport->ios_to_queue);
1051
1052 exit_qcmd:
1053         return rc;
1054 }
1055
1056 static void qedf_parse_fcp_rsp(struct qedf_ioreq *io_req,
1057                                  struct fcoe_cqe_rsp_info *fcp_rsp)
1058 {
1059         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
1060         struct qedf_ctx *qedf = io_req->fcport->qedf;
1061         u8 rsp_flags = fcp_rsp->rsp_flags.flags;
1062         int fcp_sns_len = 0;
1063         int fcp_rsp_len = 0;
1064         uint8_t *rsp_info, *sense_data;
1065
1066         io_req->fcp_status = FC_GOOD;
1067         io_req->fcp_resid = 0;
1068         if (rsp_flags & (FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER |
1069             FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER))
1070                 io_req->fcp_resid = fcp_rsp->fcp_resid;
1071
1072         io_req->scsi_comp_flags = rsp_flags;
1073         CMD_SCSI_STATUS(sc_cmd) = io_req->cdb_status =
1074             fcp_rsp->scsi_status_code;
1075
1076         if (rsp_flags &
1077             FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID)
1078                 fcp_rsp_len = fcp_rsp->fcp_rsp_len;
1079
1080         if (rsp_flags &
1081             FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID)
1082                 fcp_sns_len = fcp_rsp->fcp_sns_len;
1083
1084         io_req->fcp_rsp_len = fcp_rsp_len;
1085         io_req->fcp_sns_len = fcp_sns_len;
1086         rsp_info = sense_data = io_req->sense_buffer;
1087
1088         /* fetch fcp_rsp_code */
1089         if ((fcp_rsp_len == 4) || (fcp_rsp_len == 8)) {
1090                 /* Only for task management function */
1091                 io_req->fcp_rsp_code = rsp_info[3];
1092                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1093                     "fcp_rsp_code = %d\n", io_req->fcp_rsp_code);
1094                 /* Adjust sense-data location. */
1095                 sense_data += fcp_rsp_len;
1096         }
1097
1098         if (fcp_sns_len > SCSI_SENSE_BUFFERSIZE) {
1099                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1100                     "Truncating sense buffer\n");
1101                 fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
1102         }
1103
1104         /* The sense buffer can be NULL for TMF commands */
1105         if (sc_cmd->sense_buffer) {
1106                 memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
1107                 if (fcp_sns_len)
1108                         memcpy(sc_cmd->sense_buffer, sense_data,
1109                             fcp_sns_len);
1110         }
1111 }
1112
1113 static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
1114 {
1115         struct scsi_cmnd *sc = io_req->sc_cmd;
1116
1117         if (io_req->bd_tbl->bd_valid && sc && scsi_sg_count(sc)) {
1118                 dma_unmap_sg(&qedf->pdev->dev, scsi_sglist(sc),
1119                     scsi_sg_count(sc), sc->sc_data_direction);
1120                 io_req->bd_tbl->bd_valid = 0;
1121         }
1122 }
1123
1124 void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1125         struct qedf_ioreq *io_req)
1126 {
1127         struct scsi_cmnd *sc_cmd;
1128         struct fcoe_cqe_rsp_info *fcp_rsp;
1129         struct qedf_rport *fcport;
1130         int refcount;
1131         u16 scope, qualifier = 0;
1132         u8 fw_residual_flag = 0;
1133         unsigned long flags = 0;
1134         u16 chk_scope = 0;
1135
1136         if (!io_req)
1137                 return;
1138         if (!cqe)
1139                 return;
1140
1141         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1142             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1143             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1144                 QEDF_ERR(&qedf->dbg_ctx,
1145                          "io_req xid=0x%x already in cleanup or abort processing or already completed.\n",
1146                          io_req->xid);
1147                 return;
1148         }
1149
1150         sc_cmd = io_req->sc_cmd;
1151         fcp_rsp = &cqe->cqe_info.rsp_info;
1152
1153         if (!sc_cmd) {
1154                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1155                 return;
1156         }
1157
1158         if (!sc_cmd->SCp.ptr) {
1159                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1160                     "another context.\n");
1161                 return;
1162         }
1163
1164         if (!sc_cmd->device) {
1165                 QEDF_ERR(&qedf->dbg_ctx,
1166                          "Device for sc_cmd %p is NULL.\n", sc_cmd);
1167                 return;
1168         }
1169
1170         if (!sc_cmd->request) {
1171                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
1172                     "sc_cmd=%p.\n", sc_cmd);
1173                 return;
1174         }
1175
1176         if (!sc_cmd->request->q) {
1177                 QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request "
1178                    "is not valid, sc_cmd=%p.\n", sc_cmd);
1179                 return;
1180         }
1181
1182         fcport = io_req->fcport;
1183
1184         /*
1185          * When flush is active, let the cmds be completed from the cleanup
1186          * context
1187          */
1188         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
1189             (test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags) &&
1190              sc_cmd->device->lun == (u64)fcport->lun_reset_lun)) {
1191                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1192                           "Dropping good completion xid=0x%x as fcport is flushing",
1193                           io_req->xid);
1194                 return;
1195         }
1196
1197         qedf_parse_fcp_rsp(io_req, fcp_rsp);
1198
1199         qedf_unmap_sg_list(qedf, io_req);
1200
1201         /* Check for FCP transport error */
1202         if (io_req->fcp_rsp_len > 3 && io_req->fcp_rsp_code) {
1203                 QEDF_ERR(&(qedf->dbg_ctx),
1204                     "FCP I/O protocol failure xid=0x%x fcp_rsp_len=%d "
1205                     "fcp_rsp_code=%d.\n", io_req->xid, io_req->fcp_rsp_len,
1206                     io_req->fcp_rsp_code);
1207                 sc_cmd->result = DID_BUS_BUSY << 16;
1208                 goto out;
1209         }
1210
1211         fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
1212             FCOE_CQE_RSP_INFO_FW_UNDERRUN);
1213         if (fw_residual_flag) {
1214                 QEDF_ERR(&qedf->dbg_ctx,
1215                          "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x fcp_resid=%d fw_residual=0x%x lba=%02x%02x%02x%02x.\n",
1216                          io_req->xid, fcp_rsp->rsp_flags.flags,
1217                          io_req->fcp_resid,
1218                          cqe->cqe_info.rsp_info.fw_residual, sc_cmd->cmnd[2],
1219                          sc_cmd->cmnd[3], sc_cmd->cmnd[4], sc_cmd->cmnd[5]);
1220
1221                 if (io_req->cdb_status == 0)
1222                         sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
1223                 else
1224                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1225
1226                 /*
1227                  * Set resid to the whole buffer length so we won't try to resue
1228                  * any previously data.
1229                  */
1230                 scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1231                 goto out;
1232         }
1233
1234         switch (io_req->fcp_status) {
1235         case FC_GOOD:
1236                 if (io_req->cdb_status == 0) {
1237                         /* Good I/O completion */
1238                         sc_cmd->result = DID_OK << 16;
1239                 } else {
1240                         refcount = kref_read(&io_req->refcount);
1241                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1242                             "%d:0:%d:%lld xid=0x%0x op=0x%02x "
1243                             "lba=%02x%02x%02x%02x cdb_status=%d "
1244                             "fcp_resid=0x%x refcount=%d.\n",
1245                             qedf->lport->host->host_no, sc_cmd->device->id,
1246                             sc_cmd->device->lun, io_req->xid,
1247                             sc_cmd->cmnd[0], sc_cmd->cmnd[2], sc_cmd->cmnd[3],
1248                             sc_cmd->cmnd[4], sc_cmd->cmnd[5],
1249                             io_req->cdb_status, io_req->fcp_resid,
1250                             refcount);
1251                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1252
1253                         if (io_req->cdb_status == SAM_STAT_TASK_SET_FULL ||
1254                             io_req->cdb_status == SAM_STAT_BUSY) {
1255                                 /*
1256                                  * Check whether we need to set retry_delay at
1257                                  * all based on retry_delay module parameter
1258                                  * and the status qualifier.
1259                                  */
1260
1261                                 /* Upper 2 bits */
1262                                 scope = fcp_rsp->retry_delay_timer & 0xC000;
1263                                 /* Lower 14 bits */
1264                                 qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
1265
1266                                 if (qedf_retry_delay)
1267                                         chk_scope = 1;
1268                                 /* Record stats */
1269                                 if (io_req->cdb_status ==
1270                                     SAM_STAT_TASK_SET_FULL)
1271                                         qedf->task_set_fulls++;
1272                                 else
1273                                         qedf->busy++;
1274                         }
1275                 }
1276                 if (io_req->fcp_resid)
1277                         scsi_set_resid(sc_cmd, io_req->fcp_resid);
1278
1279                 if (chk_scope == 1) {
1280                         if ((scope == 1 || scope == 2) &&
1281                             (qualifier > 0 && qualifier <= 0x3FEF)) {
1282                                 /* Check we don't go over the max */
1283                                 if (qualifier > QEDF_RETRY_DELAY_MAX) {
1284                                         qualifier = QEDF_RETRY_DELAY_MAX;
1285                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1286                                                   "qualifier = %d\n",
1287                                                   (fcp_rsp->retry_delay_timer &
1288                                                   0x3FFF));
1289                                 }
1290                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1291                                           "Scope = %d and qualifier = %d",
1292                                           scope, qualifier);
1293                                 /*  Take fcport->rport_lock to
1294                                  *  update the retry_delay_timestamp
1295                                  */
1296                                 spin_lock_irqsave(&fcport->rport_lock, flags);
1297                                 fcport->retry_delay_timestamp =
1298                                         jiffies + (qualifier * HZ / 10);
1299                                 spin_unlock_irqrestore(&fcport->rport_lock,
1300                                                        flags);
1301
1302                         } else {
1303                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1304                                           "combination of scope = %d and qualifier = %d is not handled in qedf.\n",
1305                                           scope, qualifier);
1306                         }
1307                 }
1308                 break;
1309         default:
1310                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
1311                            io_req->fcp_status);
1312                 break;
1313         }
1314
1315 out:
1316         if (qedf_io_tracing)
1317                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
1318
1319         /*
1320          * We wait till the end of the function to clear the
1321          * outstanding bit in case we need to send an abort
1322          */
1323         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1324
1325         io_req->sc_cmd = NULL;
1326         sc_cmd->SCp.ptr =  NULL;
1327         sc_cmd->scsi_done(sc_cmd);
1328         kref_put(&io_req->refcount, qedf_release_cmd);
1329 }
1330
1331 /* Return a SCSI command in some other context besides a normal completion */
1332 void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
1333         int result)
1334 {
1335         struct scsi_cmnd *sc_cmd;
1336         int refcount;
1337
1338         if (!io_req) {
1339                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "io_req is NULL\n");
1340                 return;
1341         }
1342
1343         if (test_and_set_bit(QEDF_CMD_ERR_SCSI_DONE, &io_req->flags)) {
1344                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1345                           "io_req:%p scsi_done handling already done\n",
1346                           io_req);
1347                 return;
1348         }
1349
1350         /*
1351          * We will be done with this command after this call so clear the
1352          * outstanding bit.
1353          */
1354         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1355
1356         sc_cmd = io_req->sc_cmd;
1357
1358         if (!sc_cmd) {
1359                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1360                 return;
1361         }
1362
1363         if (!virt_addr_valid(sc_cmd)) {
1364                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd=%p is not valid.", sc_cmd);
1365                 goto bad_scsi_ptr;
1366         }
1367
1368         if (!sc_cmd->SCp.ptr) {
1369                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1370                     "another context.\n");
1371                 return;
1372         }
1373
1374         if (!sc_cmd->device) {
1375                 QEDF_ERR(&qedf->dbg_ctx, "Device for sc_cmd %p is NULL.\n",
1376                          sc_cmd);
1377                 goto bad_scsi_ptr;
1378         }
1379
1380         if (!virt_addr_valid(sc_cmd->device)) {
1381                 QEDF_ERR(&qedf->dbg_ctx,
1382                          "Device pointer for sc_cmd %p is bad.\n", sc_cmd);
1383                 goto bad_scsi_ptr;
1384         }
1385
1386         if (!sc_cmd->sense_buffer) {
1387                 QEDF_ERR(&qedf->dbg_ctx,
1388                          "sc_cmd->sense_buffer for sc_cmd %p is NULL.\n",
1389                          sc_cmd);
1390                 goto bad_scsi_ptr;
1391         }
1392
1393         if (!virt_addr_valid(sc_cmd->sense_buffer)) {
1394                 QEDF_ERR(&qedf->dbg_ctx,
1395                          "sc_cmd->sense_buffer for sc_cmd %p is bad.\n",
1396                          sc_cmd);
1397                 goto bad_scsi_ptr;
1398         }
1399
1400         if (!sc_cmd->scsi_done) {
1401                 QEDF_ERR(&qedf->dbg_ctx,
1402                          "sc_cmd->scsi_done for sc_cmd %p is NULL.\n",
1403                          sc_cmd);
1404                 goto bad_scsi_ptr;
1405         }
1406
1407         qedf_unmap_sg_list(qedf, io_req);
1408
1409         sc_cmd->result = result << 16;
1410         refcount = kref_read(&io_req->refcount);
1411         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "%d:0:%d:%lld: Completing "
1412             "sc_cmd=%p result=0x%08x op=0x%02x lba=0x%02x%02x%02x%02x, "
1413             "allowed=%d retries=%d refcount=%d.\n",
1414             qedf->lport->host->host_no, sc_cmd->device->id,
1415             sc_cmd->device->lun, sc_cmd, sc_cmd->result, sc_cmd->cmnd[0],
1416             sc_cmd->cmnd[2], sc_cmd->cmnd[3], sc_cmd->cmnd[4],
1417             sc_cmd->cmnd[5], sc_cmd->allowed, sc_cmd->retries,
1418             refcount);
1419
1420         /*
1421          * Set resid to the whole buffer length so we won't try to resue any
1422          * previously read data
1423          */
1424         scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1425
1426         if (qedf_io_tracing)
1427                 qedf_trace_io(io_req->fcport, io_req, QEDF_IO_TRACE_RSP);
1428
1429         io_req->sc_cmd = NULL;
1430         sc_cmd->SCp.ptr = NULL;
1431         sc_cmd->scsi_done(sc_cmd);
1432         kref_put(&io_req->refcount, qedf_release_cmd);
1433         return;
1434
1435 bad_scsi_ptr:
1436         /*
1437          * Clear the io_req->sc_cmd backpointer so we don't try to process
1438          * this again
1439          */
1440         io_req->sc_cmd = NULL;
1441         kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 001 */
1442 }
1443
1444 /*
1445  * Handle warning type CQE completions. This is mainly used for REC timer
1446  * popping.
1447  */
1448 void qedf_process_warning_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1449         struct qedf_ioreq *io_req)
1450 {
1451         int rval, i;
1452         struct qedf_rport *fcport = io_req->fcport;
1453         u64 err_warn_bit_map;
1454         u8 err_warn = 0xff;
1455
1456         if (!cqe) {
1457                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1458                           "cqe is NULL for io_req %p xid=0x%x\n",
1459                           io_req, io_req->xid);
1460                 return;
1461         }
1462
1463         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Warning CQE, "
1464                   "xid=0x%x\n", io_req->xid);
1465         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1466                   "err_warn_bitmap=%08x:%08x\n",
1467                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1468                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1469         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1470                   "rx_buff_off=%08x, rx_id=%04x\n",
1471                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1472                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1473                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1474
1475         /* Normalize the error bitmap value to an just an unsigned int */
1476         err_warn_bit_map = (u64)
1477             ((u64)cqe->cqe_info.err_info.err_warn_bitmap_hi << 32) |
1478             (u64)cqe->cqe_info.err_info.err_warn_bitmap_lo;
1479         for (i = 0; i < 64; i++) {
1480                 if (err_warn_bit_map & (u64)((u64)1 << i)) {
1481                         err_warn = i;
1482                         break;
1483                 }
1484         }
1485
1486         /* Check if REC TOV expired if this is a tape device */
1487         if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
1488                 if (err_warn ==
1489                     FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION) {
1490                         QEDF_ERR(&(qedf->dbg_ctx), "REC timer expired.\n");
1491                         if (!test_bit(QEDF_CMD_SRR_SENT, &io_req->flags)) {
1492                                 io_req->rx_buf_off =
1493                                     cqe->cqe_info.err_info.rx_buf_off;
1494                                 io_req->tx_buf_off =
1495                                     cqe->cqe_info.err_info.tx_buf_off;
1496                                 io_req->rx_id = cqe->cqe_info.err_info.rx_id;
1497                                 rval = qedf_send_rec(io_req);
1498                                 /*
1499                                  * We only want to abort the io_req if we
1500                                  * can't queue the REC command as we want to
1501                                  * keep the exchange open for recovery.
1502                                  */
1503                                 if (rval)
1504                                         goto send_abort;
1505                         }
1506                         return;
1507                 }
1508         }
1509
1510 send_abort:
1511         init_completion(&io_req->abts_done);
1512         rval = qedf_initiate_abts(io_req, true);
1513         if (rval)
1514                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1515 }
1516
1517 /* Cleanup a command when we receive an error detection completion */
1518 void qedf_process_error_detect(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1519         struct qedf_ioreq *io_req)
1520 {
1521         int rval;
1522
1523         if (io_req == NULL) {
1524                 QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
1525                 return;
1526         }
1527
1528         if (io_req->fcport == NULL) {
1529                 QEDF_INFO(NULL, QEDF_LOG_IO, "fcport is NULL.\n");
1530                 return;
1531         }
1532
1533         if (!cqe) {
1534                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1535                         "cqe is NULL for io_req %p\n", io_req);
1536                 return;
1537         }
1538
1539         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Error detection CQE, "
1540                   "xid=0x%x\n", io_req->xid);
1541         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1542                   "err_warn_bitmap=%08x:%08x\n",
1543                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1544                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1545         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1546                   "rx_buff_off=%08x, rx_id=%04x\n",
1547                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1548                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1549                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1550
1551         /* When flush is active, let the cmds be flushed out from the cleanup context */
1552         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &io_req->fcport->flags) ||
1553                 (test_bit(QEDF_RPORT_IN_LUN_RESET, &io_req->fcport->flags) &&
1554                  io_req->sc_cmd->device->lun == (u64)io_req->fcport->lun_reset_lun)) {
1555                 QEDF_ERR(&qedf->dbg_ctx,
1556                         "Dropping EQE for xid=0x%x as fcport is flushing",
1557                         io_req->xid);
1558                 return;
1559         }
1560
1561         if (qedf->stop_io_on_error) {
1562                 qedf_stop_all_io(qedf);
1563                 return;
1564         }
1565
1566         init_completion(&io_req->abts_done);
1567         rval = qedf_initiate_abts(io_req, true);
1568         if (rval)
1569                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1570 }
1571
1572 static void qedf_flush_els_req(struct qedf_ctx *qedf,
1573         struct qedf_ioreq *els_req)
1574 {
1575         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1576             "Flushing ELS request xid=0x%x refcount=%d.\n", els_req->xid,
1577             kref_read(&els_req->refcount));
1578
1579         /*
1580          * Need to distinguish this from a timeout when calling the
1581          * els_req->cb_func.
1582          */
1583         els_req->event = QEDF_IOREQ_EV_ELS_FLUSH;
1584
1585         clear_bit(QEDF_CMD_OUTSTANDING, &els_req->flags);
1586
1587         /* Cancel the timer */
1588         cancel_delayed_work_sync(&els_req->timeout_work);
1589
1590         /* Call callback function to complete command */
1591         if (els_req->cb_func && els_req->cb_arg) {
1592                 els_req->cb_func(els_req->cb_arg);
1593                 els_req->cb_arg = NULL;
1594         }
1595
1596         /* Release kref for original initiate_els */
1597         kref_put(&els_req->refcount, qedf_release_cmd);
1598 }
1599
1600 /* A value of -1 for lun is a wild card that means flush all
1601  * active SCSI I/Os for the target.
1602  */
1603 void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
1604 {
1605         struct qedf_ioreq *io_req;
1606         struct qedf_ctx *qedf;
1607         struct qedf_cmd_mgr *cmd_mgr;
1608         int i, rc;
1609         unsigned long flags;
1610         int flush_cnt = 0;
1611         int wait_cnt = 100;
1612         int refcount = 0;
1613
1614         if (!fcport) {
1615                 QEDF_ERR(NULL, "fcport is NULL\n");
1616                 return;
1617         }
1618
1619         /* Check that fcport is still offloaded */
1620         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1621                 QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
1622                 return;
1623         }
1624
1625         qedf = fcport->qedf;
1626
1627         if (!qedf) {
1628                 QEDF_ERR(NULL, "qedf is NULL.\n");
1629                 return;
1630         }
1631
1632         /* Only wait for all commands to be queued in the Upload context */
1633         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1634             (lun == -1)) {
1635                 while (atomic_read(&fcport->ios_to_queue)) {
1636                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1637                                   "Waiting for %d I/Os to be queued\n",
1638                                   atomic_read(&fcport->ios_to_queue));
1639                         if (wait_cnt == 0) {
1640                                 QEDF_ERR(NULL,
1641                                          "%d IOs request could not be queued\n",
1642                                          atomic_read(&fcport->ios_to_queue));
1643                         }
1644                         msleep(20);
1645                         wait_cnt--;
1646                 }
1647         }
1648
1649         cmd_mgr = qedf->cmd_mgr;
1650
1651         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1652                   "Flush active i/o's num=0x%x fcport=0x%p port_id=0x%06x scsi_id=%d.\n",
1653                   atomic_read(&fcport->num_active_ios), fcport,
1654                   fcport->rdata->ids.port_id, fcport->rport->scsi_target_id);
1655         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Locking flush mutex.\n");
1656
1657         mutex_lock(&qedf->flush_mutex);
1658         if (lun == -1) {
1659                 set_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1660         } else {
1661                 set_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1662                 fcport->lun_reset_lun = lun;
1663         }
1664
1665         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1666                 io_req = &cmd_mgr->cmds[i];
1667
1668                 if (!io_req)
1669                         continue;
1670                 if (!io_req->fcport)
1671                         continue;
1672
1673                 spin_lock_irqsave(&cmd_mgr->lock, flags);
1674
1675                 if (io_req->alloc) {
1676                         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1677                                 if (io_req->cmd_type == QEDF_SCSI_CMD)
1678                                         QEDF_ERR(&qedf->dbg_ctx,
1679                                                  "Allocated but not queued, xid=0x%x\n",
1680                                                  io_req->xid);
1681                         }
1682                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1683                 } else {
1684                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1685                         continue;
1686                 }
1687
1688                 if (io_req->fcport != fcport)
1689                         continue;
1690
1691                 /* In case of ABTS, CMD_OUTSTANDING is cleared on ABTS response,
1692                  * but RRQ is still pending.
1693                  * Workaround: Within qedf_send_rrq, we check if the fcport is
1694                  * NULL, and we drop the ref on the io_req to clean it up.
1695                  */
1696                 if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1697                         refcount = kref_read(&io_req->refcount);
1698                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1699                                   "Not outstanding, xid=0x%x, cmd_type=%d refcount=%d.\n",
1700                                   io_req->xid, io_req->cmd_type, refcount);
1701                         /* If RRQ work has been queue, try to cancel it and
1702                          * free the io_req
1703                          */
1704                         if (atomic_read(&io_req->state) ==
1705                             QEDFC_CMD_ST_RRQ_WAIT) {
1706                                 if (cancel_delayed_work_sync
1707                                     (&io_req->rrq_work)) {
1708                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1709                                                   "Putting reference for pending RRQ work xid=0x%x.\n",
1710                                                   io_req->xid);
1711                                         /* ID: 003 */
1712                                         kref_put(&io_req->refcount,
1713                                                  qedf_release_cmd);
1714                                 }
1715                         }
1716                         continue;
1717                 }
1718
1719                 /* Only consider flushing ELS during target reset */
1720                 if (io_req->cmd_type == QEDF_ELS &&
1721                     lun == -1) {
1722                         rc = kref_get_unless_zero(&io_req->refcount);
1723                         if (!rc) {
1724                                 QEDF_ERR(&(qedf->dbg_ctx),
1725                                     "Could not get kref for ELS io_req=0x%p xid=0x%x.\n",
1726                                     io_req, io_req->xid);
1727                                 continue;
1728                         }
1729                         qedf_initiate_cleanup(io_req, false);
1730                         flush_cnt++;
1731                         qedf_flush_els_req(qedf, io_req);
1732
1733                         /*
1734                          * Release the kref and go back to the top of the
1735                          * loop.
1736                          */
1737                         goto free_cmd;
1738                 }
1739
1740                 if (io_req->cmd_type == QEDF_ABTS) {
1741                         /* ID: 004 */
1742                         rc = kref_get_unless_zero(&io_req->refcount);
1743                         if (!rc) {
1744                                 QEDF_ERR(&(qedf->dbg_ctx),
1745                                     "Could not get kref for abort io_req=0x%p xid=0x%x.\n",
1746                                     io_req, io_req->xid);
1747                                 continue;
1748                         }
1749                         if (lun != -1 && io_req->lun != lun)
1750                                 goto free_cmd;
1751
1752                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1753                             "Flushing abort xid=0x%x.\n", io_req->xid);
1754
1755                         if (cancel_delayed_work_sync(&io_req->rrq_work)) {
1756                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1757                                           "Putting ref for cancelled RRQ work xid=0x%x.\n",
1758                                           io_req->xid);
1759                                 kref_put(&io_req->refcount, qedf_release_cmd);
1760                         }
1761
1762                         if (cancel_delayed_work_sync(&io_req->timeout_work)) {
1763                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1764                                           "Putting ref for cancelled tmo work xid=0x%x.\n",
1765                                           io_req->xid);
1766                                 qedf_initiate_cleanup(io_req, true);
1767                                 /* Notify eh_abort handler that ABTS is
1768                                  * complete
1769                                  */
1770                                 complete(&io_req->abts_done);
1771                                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1772                                 /* ID: 002 */
1773                                 kref_put(&io_req->refcount, qedf_release_cmd);
1774                         }
1775                         flush_cnt++;
1776                         goto free_cmd;
1777                 }
1778
1779                 if (!io_req->sc_cmd)
1780                         continue;
1781                 if (!io_req->sc_cmd->device) {
1782                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1783                                   "Device backpointer NULL for sc_cmd=%p.\n",
1784                                   io_req->sc_cmd);
1785                         /* Put reference for non-existent scsi_cmnd */
1786                         io_req->sc_cmd = NULL;
1787                         qedf_initiate_cleanup(io_req, false);
1788                         kref_put(&io_req->refcount, qedf_release_cmd);
1789                         continue;
1790                 }
1791                 if (lun > -1) {
1792                         if (io_req->lun != lun)
1793                                 continue;
1794                 }
1795
1796                 /*
1797                  * Use kref_get_unless_zero in the unlikely case the command
1798                  * we're about to flush was completed in the normal SCSI path
1799                  */
1800                 rc = kref_get_unless_zero(&io_req->refcount);
1801                 if (!rc) {
1802                         QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
1803                             "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
1804                         continue;
1805                 }
1806
1807                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1808                     "Cleanup xid=0x%x.\n", io_req->xid);
1809                 flush_cnt++;
1810
1811                 /* Cleanup task and return I/O mid-layer */
1812                 qedf_initiate_cleanup(io_req, true);
1813
1814 free_cmd:
1815                 kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 004 */
1816         }
1817
1818         wait_cnt = 60;
1819         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1820                   "Flushed 0x%x I/Os, active=0x%x.\n",
1821                   flush_cnt, atomic_read(&fcport->num_active_ios));
1822         /* Only wait for all commands to complete in the Upload context */
1823         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1824             (lun == -1)) {
1825                 while (atomic_read(&fcport->num_active_ios)) {
1826                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1827                                   "Flushed 0x%x I/Os, active=0x%x cnt=%d.\n",
1828                                   flush_cnt,
1829                                   atomic_read(&fcport->num_active_ios),
1830                                   wait_cnt);
1831                         if (wait_cnt == 0) {
1832                                 QEDF_ERR(&qedf->dbg_ctx,
1833                                          "Flushed %d I/Os, active=%d.\n",
1834                                          flush_cnt,
1835                                          atomic_read(&fcport->num_active_ios));
1836                                 for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1837                                         io_req = &cmd_mgr->cmds[i];
1838                                         if (io_req->fcport &&
1839                                             io_req->fcport == fcport) {
1840                                                 refcount =
1841                                                 kref_read(&io_req->refcount);
1842                                                 set_bit(QEDF_CMD_DIRTY,
1843                                                         &io_req->flags);
1844                                                 QEDF_ERR(&qedf->dbg_ctx,
1845                                                          "Outstanding io_req =%p xid=0x%x flags=0x%lx, sc_cmd=%p refcount=%d cmd_type=%d.\n",
1846                                                          io_req, io_req->xid,
1847                                                          io_req->flags,
1848                                                          io_req->sc_cmd,
1849                                                          refcount,
1850                                                          io_req->cmd_type);
1851                                         }
1852                                 }
1853                                 WARN_ON(1);
1854                                 break;
1855                         }
1856                         msleep(500);
1857                         wait_cnt--;
1858                 }
1859         }
1860
1861         clear_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1862         clear_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1863         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Unlocking flush mutex.\n");
1864         mutex_unlock(&qedf->flush_mutex);
1865 }
1866
1867 /*
1868  * Initiate a ABTS middle path command. Note that we don't have to initialize
1869  * the task context for an ABTS task.
1870  */
1871 int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
1872 {
1873         struct fc_lport *lport;
1874         struct qedf_rport *fcport = io_req->fcport;
1875         struct fc_rport_priv *rdata;
1876         struct qedf_ctx *qedf;
1877         u16 xid;
1878         int rc = 0;
1879         unsigned long flags;
1880         struct fcoe_wqe *sqe;
1881         u16 sqe_idx;
1882         int refcount = 0;
1883
1884         /* Sanity check qedf_rport before dereferencing any pointers */
1885         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1886                 QEDF_ERR(NULL, "tgt not offloaded\n");
1887                 rc = 1;
1888                 goto out;
1889         }
1890
1891         qedf = fcport->qedf;
1892         rdata = fcport->rdata;
1893
1894         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
1895                 QEDF_ERR(&qedf->dbg_ctx, "stale rport\n");
1896                 rc = 1;
1897                 goto out;
1898         }
1899
1900         lport = qedf->lport;
1901
1902         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
1903                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
1904                 rc = 1;
1905                 goto drop_rdata_kref;
1906         }
1907
1908         if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
1909                 QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
1910                 rc = 1;
1911                 goto drop_rdata_kref;
1912         }
1913
1914         /* Ensure room on SQ */
1915         if (!atomic_read(&fcport->free_sqes)) {
1916                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
1917                 rc = 1;
1918                 goto drop_rdata_kref;
1919         }
1920
1921         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1922                 QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
1923                 rc = 1;
1924                 goto drop_rdata_kref;
1925         }
1926
1927         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1928             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1929             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1930                 QEDF_ERR(&qedf->dbg_ctx,
1931                          "io_req xid=0x%x sc_cmd=%p already in cleanup or abort processing or already completed.\n",
1932                          io_req->xid, io_req->sc_cmd);
1933                 rc = 1;
1934                 goto drop_rdata_kref;
1935         }
1936
1937         kref_get(&io_req->refcount);
1938
1939         xid = io_req->xid;
1940         qedf->control_requests++;
1941         qedf->packet_aborts++;
1942
1943         /* Set the command type to abort */
1944         io_req->cmd_type = QEDF_ABTS;
1945         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
1946
1947         set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1948         refcount = kref_read(&io_req->refcount);
1949         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
1950                   "ABTS io_req xid = 0x%x refcount=%d\n",
1951                   xid, refcount);
1952
1953         qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT);
1954
1955         spin_lock_irqsave(&fcport->rport_lock, flags);
1956
1957         sqe_idx = qedf_get_sqe_idx(fcport);
1958         sqe = &fcport->sq[sqe_idx];
1959         memset(sqe, 0, sizeof(struct fcoe_wqe));
1960         io_req->task_params->sqe = sqe;
1961
1962         init_initiator_abort_fcoe_task(io_req->task_params);
1963         qedf_ring_doorbell(fcport);
1964
1965         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1966
1967 drop_rdata_kref:
1968         kref_put(&rdata->kref, fc_rport_destroy);
1969 out:
1970         return rc;
1971 }
1972
1973 void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1974         struct qedf_ioreq *io_req)
1975 {
1976         uint32_t r_ctl;
1977         int rc;
1978         struct qedf_rport *fcport = io_req->fcport;
1979
1980         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
1981                    "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
1982
1983         r_ctl = cqe->cqe_info.abts_info.r_ctl;
1984
1985         /* This was added at a point when we were scheduling abts_compl &
1986          * cleanup_compl on different CPUs and there was a possibility of
1987          * the io_req to be freed from the other context before we got here.
1988          */
1989         if (!fcport) {
1990                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1991                           "Dropping ABTS completion xid=0x%x as fcport is NULL",
1992                           io_req->xid);
1993                 return;
1994         }
1995
1996         /*
1997          * When flush is active, let the cmds be completed from the cleanup
1998          * context
1999          */
2000         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
2001             test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags)) {
2002                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
2003                           "Dropping ABTS completion xid=0x%x as fcport is flushing",
2004                           io_req->xid);
2005                 return;
2006         }
2007
2008         if (!cancel_delayed_work(&io_req->timeout_work)) {
2009                 QEDF_ERR(&qedf->dbg_ctx,
2010                          "Wasn't able to cancel abts timeout work.\n");
2011         }
2012
2013         switch (r_ctl) {
2014         case FC_RCTL_BA_ACC:
2015                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
2016                     "ABTS response - ACC Send RRQ after R_A_TOV\n");
2017                 io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
2018                 rc = kref_get_unless_zero(&io_req->refcount);   /* ID: 003 */
2019                 if (!rc) {
2020                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2021                                   "kref is already zero so ABTS was already completed or flushed xid=0x%x.\n",
2022                                   io_req->xid);
2023                         return;
2024                 }
2025                 /*
2026                  * Dont release this cmd yet. It will be relesed
2027                  * after we get RRQ response
2028                  */
2029                 queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
2030                     msecs_to_jiffies(qedf->lport->r_a_tov));
2031                 atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_WAIT);
2032                 break;
2033         /* For error cases let the cleanup return the command */
2034         case FC_RCTL_BA_RJT:
2035                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
2036                    "ABTS response - RJT\n");
2037                 io_req->event = QEDF_IOREQ_EV_ABORT_FAILED;
2038                 break;
2039         default:
2040                 QEDF_ERR(&(qedf->dbg_ctx), "Unknown ABTS response\n");
2041                 break;
2042         }
2043
2044         clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
2045
2046         if (io_req->sc_cmd) {
2047                 if (!io_req->return_scsi_cmd_on_abts)
2048                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2049                                   "Not call scsi_done for xid=0x%x.\n",
2050                                   io_req->xid);
2051                 if (io_req->return_scsi_cmd_on_abts)
2052                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2053         }
2054
2055         /* Notify eh_abort handler that ABTS is complete */
2056         complete(&io_req->abts_done);
2057
2058         kref_put(&io_req->refcount, qedf_release_cmd);
2059 }
2060
2061 int qedf_init_mp_req(struct qedf_ioreq *io_req)
2062 {
2063         struct qedf_mp_req *mp_req;
2064         struct scsi_sge *mp_req_bd;
2065         struct scsi_sge *mp_resp_bd;
2066         struct qedf_ctx *qedf = io_req->fcport->qedf;
2067         dma_addr_t addr;
2068         uint64_t sz;
2069
2070         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_MP_REQ, "Entered.\n");
2071
2072         mp_req = (struct qedf_mp_req *)&(io_req->mp_req);
2073         memset(mp_req, 0, sizeof(struct qedf_mp_req));
2074
2075         if (io_req->cmd_type != QEDF_ELS) {
2076                 mp_req->req_len = sizeof(struct fcp_cmnd);
2077                 io_req->data_xfer_len = mp_req->req_len;
2078         } else
2079                 mp_req->req_len = io_req->data_xfer_len;
2080
2081         mp_req->req_buf = dma_alloc_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
2082             &mp_req->req_buf_dma, GFP_KERNEL);
2083         if (!mp_req->req_buf) {
2084                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req buffer\n");
2085                 qedf_free_mp_resc(io_req);
2086                 return -ENOMEM;
2087         }
2088
2089         mp_req->resp_buf = dma_alloc_coherent(&qedf->pdev->dev,
2090             QEDF_PAGE_SIZE, &mp_req->resp_buf_dma, GFP_KERNEL);
2091         if (!mp_req->resp_buf) {
2092                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc TM resp "
2093                           "buffer\n");
2094                 qedf_free_mp_resc(io_req);
2095                 return -ENOMEM;
2096         }
2097
2098         /* Allocate and map mp_req_bd and mp_resp_bd */
2099         sz = sizeof(struct scsi_sge);
2100         mp_req->mp_req_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2101             &mp_req->mp_req_bd_dma, GFP_KERNEL);
2102         if (!mp_req->mp_req_bd) {
2103                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req bd\n");
2104                 qedf_free_mp_resc(io_req);
2105                 return -ENOMEM;
2106         }
2107
2108         mp_req->mp_resp_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2109             &mp_req->mp_resp_bd_dma, GFP_KERNEL);
2110         if (!mp_req->mp_resp_bd) {
2111                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP resp bd\n");
2112                 qedf_free_mp_resc(io_req);
2113                 return -ENOMEM;
2114         }
2115
2116         /* Fill bd table */
2117         addr = mp_req->req_buf_dma;
2118         mp_req_bd = mp_req->mp_req_bd;
2119         mp_req_bd->sge_addr.lo = U64_LO(addr);
2120         mp_req_bd->sge_addr.hi = U64_HI(addr);
2121         mp_req_bd->sge_len = QEDF_PAGE_SIZE;
2122
2123         /*
2124          * MP buffer is either a task mgmt command or an ELS.
2125          * So the assumption is that it consumes a single bd
2126          * entry in the bd table
2127          */
2128         mp_resp_bd = mp_req->mp_resp_bd;
2129         addr = mp_req->resp_buf_dma;
2130         mp_resp_bd->sge_addr.lo = U64_LO(addr);
2131         mp_resp_bd->sge_addr.hi = U64_HI(addr);
2132         mp_resp_bd->sge_len = QEDF_PAGE_SIZE;
2133
2134         return 0;
2135 }
2136
2137 /*
2138  * Last ditch effort to clear the port if it's stuck. Used only after a
2139  * cleanup task times out.
2140  */
2141 static void qedf_drain_request(struct qedf_ctx *qedf)
2142 {
2143         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
2144                 QEDF_ERR(&(qedf->dbg_ctx), "MCP drain already active.\n");
2145                 return;
2146         }
2147
2148         /* Set bit to return all queuecommand requests as busy */
2149         set_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2150
2151         /* Call qed drain request for function. Should be synchronous */
2152         qed_ops->common->drain(qedf->cdev);
2153
2154         /* Settle time for CQEs to be returned */
2155         msleep(100);
2156
2157         /* Unplug and continue */
2158         clear_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2159 }
2160
2161 /*
2162  * Returns SUCCESS if the cleanup task does not timeout, otherwise return
2163  * FAILURE.
2164  */
2165 int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
2166         bool return_scsi_cmd_on_abts)
2167 {
2168         struct qedf_rport *fcport;
2169         struct qedf_ctx *qedf;
2170         int tmo = 0;
2171         int rc = SUCCESS;
2172         unsigned long flags;
2173         struct fcoe_wqe *sqe;
2174         u16 sqe_idx;
2175         int refcount = 0;
2176
2177         fcport = io_req->fcport;
2178         if (!fcport) {
2179                 QEDF_ERR(NULL, "fcport is NULL.\n");
2180                 return SUCCESS;
2181         }
2182
2183         /* Sanity check qedf_rport before dereferencing any pointers */
2184         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2185                 QEDF_ERR(NULL, "tgt not offloaded\n");
2186                 return SUCCESS;
2187         }
2188
2189         qedf = fcport->qedf;
2190         if (!qedf) {
2191                 QEDF_ERR(NULL, "qedf is NULL.\n");
2192                 return SUCCESS;
2193         }
2194
2195         if (io_req->cmd_type == QEDF_ELS) {
2196                 goto process_els;
2197         }
2198
2199         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
2200             test_and_set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
2201                 QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
2202                           "cleanup processing or already completed.\n",
2203                           io_req->xid);
2204                 return SUCCESS;
2205         }
2206         set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2207
2208 process_els:
2209         /* Ensure room on SQ */
2210         if (!atomic_read(&fcport->free_sqes)) {
2211                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
2212                 /* Need to make sure we clear the flag since it was set */
2213                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2214                 return FAILED;
2215         }
2216
2217         if (io_req->cmd_type == QEDF_CLEANUP) {
2218                 QEDF_ERR(&qedf->dbg_ctx,
2219                          "io_req=0x%x is already a cleanup command cmd_type=%d.\n",
2220                          io_req->xid, io_req->cmd_type);
2221                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2222                 return SUCCESS;
2223         }
2224
2225         refcount = kref_read(&io_req->refcount);
2226
2227         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
2228                   "Entered xid=0x%x sc_cmd=%p cmd_type=%d flags=0x%lx refcount=%d fcport=%p port_id=0x%06x\n",
2229                   io_req->xid, io_req->sc_cmd, io_req->cmd_type, io_req->flags,
2230                   refcount, fcport, fcport->rdata->ids.port_id);
2231
2232         /* Cleanup cmds re-use the same TID as the original I/O */
2233         io_req->cmd_type = QEDF_CLEANUP;
2234         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
2235
2236         init_completion(&io_req->cleanup_done);
2237
2238         spin_lock_irqsave(&fcport->rport_lock, flags);
2239
2240         sqe_idx = qedf_get_sqe_idx(fcport);
2241         sqe = &fcport->sq[sqe_idx];
2242         memset(sqe, 0, sizeof(struct fcoe_wqe));
2243         io_req->task_params->sqe = sqe;
2244
2245         init_initiator_cleanup_fcoe_task(io_req->task_params);
2246         qedf_ring_doorbell(fcport);
2247
2248         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2249
2250         tmo = wait_for_completion_timeout(&io_req->cleanup_done,
2251                                           QEDF_CLEANUP_TIMEOUT * HZ);
2252
2253         if (!tmo) {
2254                 rc = FAILED;
2255                 /* Timeout case */
2256                 QEDF_ERR(&(qedf->dbg_ctx), "Cleanup command timeout, "
2257                           "xid=%x.\n", io_req->xid);
2258                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2259                 /* Issue a drain request if cleanup task times out */
2260                 QEDF_ERR(&(qedf->dbg_ctx), "Issuing MCP drain request.\n");
2261                 qedf_drain_request(qedf);
2262         }
2263
2264         /* If it TASK MGMT handle it, reference will be decreased
2265          * in qedf_execute_tmf
2266          */
2267         if (io_req->tm_flags  == FCP_TMF_LUN_RESET ||
2268             io_req->tm_flags == FCP_TMF_TGT_RESET) {
2269                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2270                 io_req->sc_cmd = NULL;
2271                 kref_put(&io_req->refcount, qedf_release_cmd);
2272                 complete(&io_req->tm_done);
2273         }
2274
2275         if (io_req->sc_cmd) {
2276                 if (!io_req->return_scsi_cmd_on_abts)
2277                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2278                                   "Not call scsi_done for xid=0x%x.\n",
2279                                   io_req->xid);
2280                 if (io_req->return_scsi_cmd_on_abts)
2281                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2282         }
2283
2284         if (rc == SUCCESS)
2285                 io_req->event = QEDF_IOREQ_EV_CLEANUP_SUCCESS;
2286         else
2287                 io_req->event = QEDF_IOREQ_EV_CLEANUP_FAILED;
2288
2289         return rc;
2290 }
2291
2292 void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2293         struct qedf_ioreq *io_req)
2294 {
2295         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid = 0x%x\n",
2296                    io_req->xid);
2297
2298         clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2299
2300         /* Complete so we can finish cleaning up the I/O */
2301         complete(&io_req->cleanup_done);
2302 }
2303
2304 static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
2305         uint8_t tm_flags)
2306 {
2307         struct qedf_ioreq *io_req;
2308         struct e4_fcoe_task_context *task;
2309         struct qedf_ctx *qedf = fcport->qedf;
2310         struct fc_lport *lport = qedf->lport;
2311         int rc = 0;
2312         uint16_t xid;
2313         int tmo = 0;
2314         int lun = 0;
2315         unsigned long flags;
2316         struct fcoe_wqe *sqe;
2317         u16 sqe_idx;
2318
2319         if (!sc_cmd) {
2320                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd is NULL\n");
2321                 return FAILED;
2322         }
2323
2324         lun = (int)sc_cmd->device->lun;
2325         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2326                 QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
2327                 rc = FAILED;
2328                 goto no_flush;
2329         }
2330
2331         io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
2332         if (!io_req) {
2333                 QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
2334                 rc = -EAGAIN;
2335                 goto no_flush;
2336         }
2337
2338         if (tm_flags == FCP_TMF_LUN_RESET)
2339                 qedf->lun_resets++;
2340         else if (tm_flags == FCP_TMF_TGT_RESET)
2341                 qedf->target_resets++;
2342
2343         /* Initialize rest of io_req fields */
2344         io_req->sc_cmd = sc_cmd;
2345         io_req->fcport = fcport;
2346         io_req->cmd_type = QEDF_TASK_MGMT_CMD;
2347
2348         /* Record which cpu this request is associated with */
2349         io_req->cpu = smp_processor_id();
2350
2351         /* Set TM flags */
2352         io_req->io_req_flags = QEDF_READ;
2353         io_req->data_xfer_len = 0;
2354         io_req->tm_flags = tm_flags;
2355
2356         /* Default is to return a SCSI command when an error occurs */
2357         io_req->return_scsi_cmd_on_abts = false;
2358
2359         /* Obtain exchange id */
2360         xid = io_req->xid;
2361
2362         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "TMF io_req xid = "
2363                    "0x%x\n", xid);
2364
2365         /* Initialize task context for this IO request */
2366         task = qedf_get_task_mem(&qedf->tasks, xid);
2367
2368         init_completion(&io_req->tm_done);
2369
2370         spin_lock_irqsave(&fcport->rport_lock, flags);
2371
2372         sqe_idx = qedf_get_sqe_idx(fcport);
2373         sqe = &fcport->sq[sqe_idx];
2374         memset(sqe, 0, sizeof(struct fcoe_wqe));
2375
2376         qedf_init_task(fcport, lport, io_req, task, sqe);
2377         qedf_ring_doorbell(fcport);
2378
2379         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2380
2381         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2382         tmo = wait_for_completion_timeout(&io_req->tm_done,
2383             QEDF_TM_TIMEOUT * HZ);
2384
2385         if (!tmo) {
2386                 rc = FAILED;
2387                 QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
2388                 /* Clear outstanding bit since command timed out */
2389                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2390                 io_req->sc_cmd = NULL;
2391         } else {
2392                 /* Check TMF response code */
2393                 if (io_req->fcp_rsp_code == 0)
2394                         rc = SUCCESS;
2395                 else
2396                         rc = FAILED;
2397         }
2398         /*
2399          * Double check that fcport has not gone into an uploading state before
2400          * executing the command flush for the LUN/target.
2401          */
2402         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2403                 QEDF_ERR(&qedf->dbg_ctx,
2404                          "fcport is uploading, not executing flush.\n");
2405                 goto no_flush;
2406         }
2407         /* We do not need this io_req any more */
2408         kref_put(&io_req->refcount, qedf_release_cmd);
2409
2410
2411         if (tm_flags == FCP_TMF_LUN_RESET)
2412                 qedf_flush_active_ios(fcport, lun);
2413         else
2414                 qedf_flush_active_ios(fcport, -1);
2415
2416 no_flush:
2417         if (rc != SUCCESS) {
2418                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
2419                 rc = FAILED;
2420         } else {
2421                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
2422                 rc = SUCCESS;
2423         }
2424         return rc;
2425 }
2426
2427 int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
2428 {
2429         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
2430         struct fc_rport_libfc_priv *rp = rport->dd_data;
2431         struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
2432         struct qedf_ctx *qedf;
2433         struct fc_lport *lport = shost_priv(sc_cmd->device->host);
2434         int rc = SUCCESS;
2435         int rval;
2436         struct qedf_ioreq *io_req = NULL;
2437         int ref_cnt = 0;
2438         struct fc_rport_priv *rdata = fcport->rdata;
2439
2440         QEDF_ERR(NULL,
2441                  "tm_flags 0x%x sc_cmd %p op = 0x%02x target_id = 0x%x lun=%d\n",
2442                  tm_flags, sc_cmd, sc_cmd->cmd_len ? sc_cmd->cmnd[0] : 0xff,
2443                  rport->scsi_target_id, (int)sc_cmd->device->lun);
2444
2445         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
2446                 QEDF_ERR(NULL, "stale rport\n");
2447                 return FAILED;
2448         }
2449
2450         QEDF_ERR(NULL, "portid=%06x tm_flags =%s\n", rdata->ids.port_id,
2451                  (tm_flags == FCP_TMF_TGT_RESET) ? "TARGET RESET" :
2452                  "LUN RESET");
2453
2454         if (sc_cmd->SCp.ptr) {
2455                 io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
2456                 ref_cnt = kref_read(&io_req->refcount);
2457                 QEDF_ERR(NULL,
2458                          "orig io_req = %p xid = 0x%x ref_cnt = %d.\n",
2459                          io_req, io_req->xid, ref_cnt);
2460         }
2461
2462         rval = fc_remote_port_chkready(rport);
2463         if (rval) {
2464                 QEDF_ERR(NULL, "device_reset rport not ready\n");
2465                 rc = FAILED;
2466                 goto tmf_err;
2467         }
2468
2469         rc = fc_block_scsi_eh(sc_cmd);
2470         if (rc)
2471                 goto tmf_err;
2472
2473         if (!fcport) {
2474                 QEDF_ERR(NULL, "device_reset: rport is NULL\n");
2475                 rc = FAILED;
2476                 goto tmf_err;
2477         }
2478
2479         qedf = fcport->qedf;
2480
2481         if (!qedf) {
2482                 QEDF_ERR(NULL, "qedf is NULL.\n");
2483                 rc = FAILED;
2484                 goto tmf_err;
2485         }
2486
2487         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2488                 QEDF_ERR(&qedf->dbg_ctx, "Connection is getting uploaded.\n");
2489                 rc = SUCCESS;
2490                 goto tmf_err;
2491         }
2492
2493         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
2494             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
2495                 rc = SUCCESS;
2496                 goto tmf_err;
2497         }
2498
2499         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
2500                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
2501                 rc = FAILED;
2502                 goto tmf_err;
2503         }
2504
2505         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2506                 if (!fcport->rdata)
2507                         QEDF_ERR(&qedf->dbg_ctx, "fcport %p is uploading.\n",
2508                                  fcport);
2509                 else
2510                         QEDF_ERR(&qedf->dbg_ctx,
2511                                  "fcport %p port_id=%06x is uploading.\n",
2512                                  fcport, fcport->rdata->ids.port_id);
2513                 rc = FAILED;
2514                 goto tmf_err;
2515         }
2516
2517         rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
2518
2519 tmf_err:
2520         kref_put(&rdata->kref, fc_rport_destroy);
2521         return rc;
2522 }
2523
2524 void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2525         struct qedf_ioreq *io_req)
2526 {
2527         struct fcoe_cqe_rsp_info *fcp_rsp;
2528
2529         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2530
2531         fcp_rsp = &cqe->cqe_info.rsp_info;
2532         qedf_parse_fcp_rsp(io_req, fcp_rsp);
2533
2534         io_req->sc_cmd = NULL;
2535         complete(&io_req->tm_done);
2536 }
2537
2538 void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
2539         struct fcoe_cqe *cqe)
2540 {
2541         unsigned long flags;
2542         uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
2543         u32 payload_len, crc;
2544         struct fc_frame_header *fh;
2545         struct fc_frame *fp;
2546         struct qedf_io_work *io_work;
2547         u32 bdq_idx;
2548         void *bdq_addr;
2549         struct scsi_bd *p_bd_info;
2550
2551         p_bd_info = &cqe->cqe_info.unsolic_info.bd_info;
2552         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2553                   "address.hi=%x, address.lo=%x, opaque_data.hi=%x, opaque_data.lo=%x, bdq_prod_idx=%u, len=%u\n",
2554                   le32_to_cpu(p_bd_info->address.hi),
2555                   le32_to_cpu(p_bd_info->address.lo),
2556                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.hi),
2557                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo),
2558                   qedf->bdq_prod_idx, pktlen);
2559
2560         bdq_idx = le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo);
2561         if (bdq_idx >= QEDF_BDQ_SIZE) {
2562                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_idx is out of range %d.\n",
2563                     bdq_idx);
2564                 goto increment_prod;
2565         }
2566
2567         bdq_addr = qedf->bdq[bdq_idx].buf_addr;
2568         if (!bdq_addr) {
2569                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_addr is NULL, dropping "
2570                     "unsolicited packet.\n");
2571                 goto increment_prod;
2572         }
2573
2574         if (qedf_dump_frames) {
2575                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2576                     "BDQ frame is at addr=%p.\n", bdq_addr);
2577                 print_hex_dump(KERN_WARNING, "bdq ", DUMP_PREFIX_OFFSET, 16, 1,
2578                     (void *)bdq_addr, pktlen, false);
2579         }
2580
2581         /* Allocate frame */
2582         payload_len = pktlen - sizeof(struct fc_frame_header);
2583         fp = fc_frame_alloc(qedf->lport, payload_len);
2584         if (!fp) {
2585                 QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate fp.\n");
2586                 goto increment_prod;
2587         }
2588
2589         /* Copy data from BDQ buffer into fc_frame struct */
2590         fh = (struct fc_frame_header *)fc_frame_header_get(fp);
2591         memcpy(fh, (void *)bdq_addr, pktlen);
2592
2593         QEDF_WARN(&qedf->dbg_ctx,
2594                   "Processing Unsolicated frame, src=%06x dest=%06x r_ctl=0x%x type=0x%x cmd=%02x\n",
2595                   ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id), fh->fh_r_ctl,
2596                   fh->fh_type, fc_frame_payload_op(fp));
2597
2598         /* Initialize the frame so libfc sees it as a valid frame */
2599         crc = fcoe_fc_crc(fp);
2600         fc_frame_init(fp);
2601         fr_dev(fp) = qedf->lport;
2602         fr_sof(fp) = FC_SOF_I3;
2603         fr_eof(fp) = FC_EOF_T;
2604         fr_crc(fp) = cpu_to_le32(~crc);
2605
2606         /*
2607          * We need to return the frame back up to libfc in a non-atomic
2608          * context
2609          */
2610         io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
2611         if (!io_work) {
2612                 QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
2613                            "work for I/O completion.\n");
2614                 fc_frame_free(fp);
2615                 goto increment_prod;
2616         }
2617         memset(io_work, 0, sizeof(struct qedf_io_work));
2618
2619         INIT_WORK(&io_work->work, qedf_fp_io_handler);
2620
2621         /* Copy contents of CQE for deferred processing */
2622         memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
2623
2624         io_work->qedf = qedf;
2625         io_work->fp = fp;
2626
2627         queue_work_on(smp_processor_id(), qedf_io_wq, &io_work->work);
2628 increment_prod:
2629         spin_lock_irqsave(&qedf->hba_lock, flags);
2630
2631         /* Increment producer to let f/w know we've handled the frame */
2632         qedf->bdq_prod_idx++;
2633
2634         /* Producer index wraps at uint16_t boundary */
2635         if (qedf->bdq_prod_idx == 0xffff)
2636                 qedf->bdq_prod_idx = 0;
2637
2638         writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
2639         readw(qedf->bdq_primary_prod);
2640         writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
2641         readw(qedf->bdq_secondary_prod);
2642
2643         spin_unlock_irqrestore(&qedf->hba_lock, flags);
2644 }