drivers/scsi/elx/efct/efct_scsi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
   4  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
   5  */
   6
   7 #include "efct_driver.h"
   8 #include "efct_hw.h"
   9
  10 #define enable_tsend_auto_resp(efct)    1
  11 #define enable_treceive_auto_resp(efct) 0
  12
  13 #define SCSI_IOFMT "[%04x][i:%04x t:%04x h:%04x]"
  14
  15 #define scsi_io_printf(io, fmt, ...) \
  16         efc_log_debug(io->efct, "[%s]" SCSI_IOFMT fmt, \
  17                 io->node->display_name, io->instance_index,\
  18                 io->init_task_tag, io->tgt_task_tag, io->hw_tag, ##__VA_ARGS__)
  19
  20 #define EFCT_LOG_ENABLE_SCSI_TRACE(efct)                \
  21                 (((efct) != NULL) ? (((efct)->logmask & (1U << 2)) != 0) : 0)
  22
  23 #define scsi_io_trace(io, fmt, ...) \
  24         do { \
  25                 if (EFCT_LOG_ENABLE_SCSI_TRACE(io->efct)) \
  26                         scsi_io_printf(io, fmt, ##__VA_ARGS__); \
  27         } while (0)
  28
  29 struct efct_io *
  30 efct_scsi_io_alloc(struct efct_node *node)
  31 {
  32         struct efct *efct;
  33         struct efct_xport *xport;
  34         struct efct_io *io;
  35         unsigned long flags;
  36
  37         efct = node->efct;
  38
  39         xport = efct->xport;
  40
  41         io = efct_io_pool_io_alloc(efct->xport->io_pool);
  42         if (!io) {
  43                 efc_log_err(efct, "IO alloc Failed\n");
  44                 atomic_add_return(1, &xport->io_alloc_failed_count);
  45                 return NULL;
  46         }
  47
  48         /* initialize refcount */
  49         kref_init(&io->ref);
  50         io->release = _efct_scsi_io_free;
  51
  52         /* set generic fields */
  53         io->efct = efct;
  54         io->node = node;
  55         kref_get(&node->ref);
  56
  57         /* set type and name */
  58         io->io_type = EFCT_IO_TYPE_IO;
  59         io->display_name = "scsi_io";
  60
  61         io->cmd_ini = false;
  62         io->cmd_tgt = true;
  63
  64         /* Add to node's active_ios list */
  65         INIT_LIST_HEAD(&io->list_entry);
  66         spin_lock_irqsave(&node->active_ios_lock, flags);
  67         list_add(&io->list_entry, &node->active_ios);
  68
  69         spin_unlock_irqrestore(&node->active_ios_lock, flags);
  70
  71         return io;
  72 }
  73
  74 void
  75 _efct_scsi_io_free(struct kref *arg)
  76 {
  77         struct efct_io *io = container_of(arg, struct efct_io, ref);
  78         struct efct *efct = io->efct;
  79         struct efct_node *node = io->node;
  80         unsigned long flags = 0;
  81
  82         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
  83
  84         if (io->io_free) {
  85                 efc_log_err(efct, "IO already freed.\n");
  86                 return;
  87         }
  88
  89         spin_lock_irqsave(&node->active_ios_lock, flags);
  90         list_del_init(&io->list_entry);
  91         spin_unlock_irqrestore(&node->active_ios_lock, flags);
  92
  93         kref_put(&node->ref, node->release);
  94         io->node = NULL;
  95         efct_io_pool_io_free(efct->xport->io_pool, io);
  96 }
  97
  98 void
  99 efct_scsi_io_free(struct efct_io *io)
 100 {
 101         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
 102         WARN_ON(!refcount_read(&io->ref.refcount));
 103         kref_put(&io->ref, io->release);
 104 }
 105
 106 static void
 107 efct_target_io_cb(struct efct_hw_io *hio, u32 length, int status,
 108                   u32 ext_status, void *app)
 109 {
 110         u32 flags = 0;
 111         struct efct_io *io = app;
 112         struct efct *efct;
 113         enum efct_scsi_io_status scsi_stat = EFCT_SCSI_STATUS_GOOD;
 114         efct_scsi_io_cb_t cb;
 115
 116         if (!io || !io->efct) {
 117                 pr_err("%s: IO can not be NULL\n", __func__);
 118                 return;
 119         }
 120
 121         scsi_io_trace(io, "status x%x ext_status x%x\n", status, ext_status);
 122
 123         efct = io->efct;
 124
 125         io->transferred += length;
 126
 127         if (!io->scsi_tgt_cb) {
 128                 efct_scsi_check_pending(efct);
 129                 return;
 130         }
 131
 132         /* Call target server completion */
 133         cb = io->scsi_tgt_cb;
 134
 135         /* Clear the callback before invoking the callback */
 136         io->scsi_tgt_cb = NULL;
 137
 138         /* if status was good, and auto-good-response was set,
 139          * then callback target-server with IO_CMPL_RSP_SENT,
 140          * otherwise send IO_CMPL
 141          */
 142         if (status == 0 && io->auto_resp)
 143                 flags |= EFCT_SCSI_IO_CMPL_RSP_SENT;
 144         else
 145                 flags |= EFCT_SCSI_IO_CMPL;
 146
 147         switch (status) {
 148         case SLI4_FC_WCQE_STATUS_SUCCESS:
 149                 scsi_stat = EFCT_SCSI_STATUS_GOOD;
 150                 break;
 151         case SLI4_FC_WCQE_STATUS_DI_ERROR:
 152                 if (ext_status & SLI4_FC_DI_ERROR_GE)
 153                         scsi_stat = EFCT_SCSI_STATUS_DIF_GUARD_ERR;
 154                 else if (ext_status & SLI4_FC_DI_ERROR_AE)
 155                         scsi_stat = EFCT_SCSI_STATUS_DIF_APP_TAG_ERROR;
 156                 else if (ext_status & SLI4_FC_DI_ERROR_RE)
 157                         scsi_stat = EFCT_SCSI_STATUS_DIF_REF_TAG_ERROR;
 158                 else
 159                         scsi_stat = EFCT_SCSI_STATUS_DIF_UNKNOWN_ERROR;
 160                 break;
 161         case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
 162                 switch (ext_status) {
 163                 case SLI4_FC_LOCAL_REJECT_INVALID_RELOFFSET:
 164                 case SLI4_FC_LOCAL_REJECT_ABORT_REQUESTED:
 165                         scsi_stat = EFCT_SCSI_STATUS_ABORTED;
 166                         break;
 167                 case SLI4_FC_LOCAL_REJECT_INVALID_RPI:
 168                         scsi_stat = EFCT_SCSI_STATUS_NEXUS_LOST;
 169                         break;
 170                 case SLI4_FC_LOCAL_REJECT_NO_XRI:
 171                         scsi_stat = EFCT_SCSI_STATUS_NO_IO;
 172                         break;
 173                 default:
 174                         /*we have seen 0x0d(TX_DMA_FAILED err)*/
 175                         scsi_stat = EFCT_SCSI_STATUS_ERROR;
 176                         break;
 177                 }
 178                 break;
 179
 180         case SLI4_FC_WCQE_STATUS_TARGET_WQE_TIMEOUT:
 181                 /* target IO timed out */
 182                 scsi_stat = EFCT_SCSI_STATUS_TIMEDOUT_AND_ABORTED;
 183                 break;
 184
 185         case SLI4_FC_WCQE_STATUS_SHUTDOWN:
 186                 /* Target IO cancelled by HW */
 187                 scsi_stat = EFCT_SCSI_STATUS_SHUTDOWN;
 188                 break;
 189
 190         default:
 191                 scsi_stat = EFCT_SCSI_STATUS_ERROR;
 192                 break;
 193         }
 194
 195         cb(io, scsi_stat, flags, io->scsi_tgt_cb_arg);
 196
 197         efct_scsi_check_pending(efct);
 198 }
 199
 200 static int
 201 efct_scsi_build_sgls(struct efct_hw *hw, struct efct_hw_io *hio,
 202                      struct efct_scsi_sgl *sgl, u32 sgl_count,
 203                      enum efct_hw_io_type type)
 204 {
 205         int rc;
 206         u32 i;
 207         struct efct *efct = hw->os;
 208
 209         /* Initialize HW SGL */
 210         rc = efct_hw_io_init_sges(hw, hio, type);
 211         if (rc) {
 212                 efc_log_err(efct, "efct_hw_io_init_sges failed: %d\n", rc);
 213                 return -EIO;
 214         }
 215
 216         for (i = 0; i < sgl_count; i++) {
 217                 /* Add data SGE */
 218                 rc = efct_hw_io_add_sge(hw, hio, sgl[i].addr, sgl[i].len);
 219                 if (rc) {
 220                         efc_log_err(efct, "add sge failed cnt=%d rc=%d\n",
 221                                     sgl_count, rc);
 222                         return rc;
 223                 }
 224         }
 225
 226         return 0;
 227 }
 228
 229 static void efc_log_sgl(struct efct_io *io)
 230 {
 231         struct efct_hw_io *hio = io->hio;
 232         struct sli4_sge *data = NULL;
 233         u32 *dword = NULL;
 234         u32 i;
 235         u32 n_sge;
 236
 237         scsi_io_trace(io, "def_sgl at 0x%x 0x%08x\n",
 238                       upper_32_bits(hio->def_sgl.phys),
 239                       lower_32_bits(hio->def_sgl.phys));
 240         n_sge = (hio->sgl == &hio->def_sgl) ? hio->n_sge : hio->def_sgl_count;
 241         for (i = 0, data = hio->def_sgl.virt; i < n_sge; i++, data++) {
 242                 dword = (u32 *)data;
 243
 244                 scsi_io_trace(io, "SGL %2d 0x%08x 0x%08x 0x%08x 0x%08x\n",
 245                               i, dword[0], dword[1], dword[2], dword[3]);
 246
 247                 if (dword[2] & (1U << 31))
 248                         break;
 249         }
 250 }
 251
 252 static void
 253 efct_scsi_check_pending_async_cb(struct efct_hw *hw, int status,
 254                                  u8 *mqe, void *arg)
 255 {
 256         struct efct_io *io = arg;
 257
 258         if (io) {
 259                 efct_hw_done_t cb = io->hw_cb;
 260
 261                 if (!io->hw_cb)
 262                         return;
 263
 264                 io->hw_cb = NULL;
 265                 (cb)(io->hio, 0, SLI4_FC_WCQE_STATUS_DISPATCH_ERROR, 0, io);
 266         }
 267 }
 268
 269 static int
 270 efct_scsi_io_dispatch_hw_io(struct efct_io *io, struct efct_hw_io *hio)
 271 {
 272         int rc = 0;
 273         struct efct *efct = io->efct;
 274
 275         /* Got a HW IO;
 276          * update ini/tgt_task_tag with HW IO info and dispatch
 277          */
 278         io->hio = hio;
 279         if (io->cmd_tgt)
 280                 io->tgt_task_tag = hio->indicator;
 281         else if (io->cmd_ini)
 282                 io->init_task_tag = hio->indicator;
 283         io->hw_tag = hio->reqtag;
 284
 285         hio->eq = io->hw_priv;
 286
 287         /* Copy WQ steering */
 288         switch (io->wq_steering) {
 289         case EFCT_SCSI_WQ_STEERING_CLASS >> EFCT_SCSI_WQ_STEERING_SHIFT:
 290                 hio->wq_steering = EFCT_HW_WQ_STEERING_CLASS;
 291                 break;
 292         case EFCT_SCSI_WQ_STEERING_REQUEST >> EFCT_SCSI_WQ_STEERING_SHIFT:
 293                 hio->wq_steering = EFCT_HW_WQ_STEERING_REQUEST;
 294                 break;
 295         case EFCT_SCSI_WQ_STEERING_CPU >> EFCT_SCSI_WQ_STEERING_SHIFT:
 296                 hio->wq_steering = EFCT_HW_WQ_STEERING_CPU;
 297                 break;
 298         }
 299
 300         switch (io->io_type) {
 301         case EFCT_IO_TYPE_IO:
 302                 rc = efct_scsi_build_sgls(&efct->hw, io->hio,
 303                                           io->sgl, io->sgl_count, io->hio_type);
 304                 if (rc)
 305                         break;
 306
 307                 if (EFCT_LOG_ENABLE_SCSI_TRACE(efct))
 308                         efc_log_sgl(io);
 309
 310                 if (io->app_id)
 311                         io->iparam.fcp_tgt.app_id = io->app_id;
 312
 313                 io->iparam.fcp_tgt.vpi = io->node->vpi;
 314                 io->iparam.fcp_tgt.rpi = io->node->rpi;
 315                 io->iparam.fcp_tgt.s_id = io->node->port_fc_id;
 316                 io->iparam.fcp_tgt.d_id = io->node->node_fc_id;
 317                 io->iparam.fcp_tgt.xmit_len = io->wire_len;
 318
 319                 rc = efct_hw_io_send(&io->efct->hw, io->hio_type, io->hio,
 320                                      &io->iparam, io->hw_cb, io);
 321                 break;
 322         default:
 323                 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 324                 rc = -EIO;
 325                 break;
 326         }
 327         return rc;
 328 }
 329
 330 static int
 331 efct_scsi_io_dispatch_no_hw_io(struct efct_io *io)
 332 {
 333         int rc;
 334
 335         switch (io->io_type) {
 336         case EFCT_IO_TYPE_ABORT: {
 337                 struct efct_hw_io *hio_to_abort = NULL;
 338
 339                 hio_to_abort = io->io_to_abort->hio;
 340
 341                 if (!hio_to_abort) {
 342                         /*
 343                          * If "IO to abort" does not have an
 344                          * associated HW IO, immediately make callback with
 345                          * success. The command must have been sent to
 346                          * the backend, but the data phase has not yet
 347                          * started, so we don't have a HW IO.
 348                          *
 349                          * Note: since the backend shims should be
 350                          * taking a reference on io_to_abort, it should not
 351                          * be possible to have been completed and freed by
 352                          * the backend before the abort got here.
 353                          */
 354                         scsi_io_printf(io, "IO: not active\n");
 355                         ((efct_hw_done_t)io->hw_cb)(io->hio, 0,
 356                                         SLI4_FC_WCQE_STATUS_SUCCESS, 0, io);
 357                         rc = 0;
 358                         break;
 359                 }
 360
 361                 /* HW IO is valid, abort it */
 362                 scsi_io_printf(io, "aborting\n");
 363                 rc = efct_hw_io_abort(&io->efct->hw, hio_to_abort,
 364                                       io->send_abts, io->hw_cb, io);
 365                 if (rc) {
 366                         int status = SLI4_FC_WCQE_STATUS_SUCCESS;
 367                         efct_hw_done_t cb = io->hw_cb;
 368
 369                         if (rc != -ENOENT && rc != -EINPROGRESS) {
 370                                 status = -1;
 371                                 scsi_io_printf(io, "Failed to abort IO rc=%d\n",
 372                                                rc);
 373                         }
 374                         cb(io->hio, 0, status, 0, io);
 375                         rc = 0;
 376                 }
 377
 378                 break;
 379         }
 380         default:
 381                 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 382                 rc = -EIO;
 383                 break;
 384         }
 385         return rc;
 386 }
 387
 388 static struct efct_io *
 389 efct_scsi_dispatch_pending(struct efct *efct)
 390 {
 391         struct efct_xport *xport = efct->xport;
 392         struct efct_io *io = NULL;
 393         struct efct_hw_io *hio;
 394         unsigned long flags = 0;
 395         int status;
 396
 397         spin_lock_irqsave(&xport->io_pending_lock, flags);
 398
 399         if (!list_empty(&xport->io_pending_list)) {
 400                 io = list_first_entry(&xport->io_pending_list, struct efct_io,
 401                                       io_pending_link);
 402                 list_del_init(&io->io_pending_link);
 403         }
 404
 405         if (!io) {
 406                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 407                 return NULL;
 408         }
 409
 410         if (io->io_type == EFCT_IO_TYPE_ABORT) {
 411                 hio = NULL;
 412         } else {
 413                 hio = efct_hw_io_alloc(&efct->hw);
 414                 if (!hio) {
 415                         /*
 416                          * No HW IO available.Put IO back on
 417                          * the front of pending list
 418                          */
 419                         list_add(&xport->io_pending_list, &io->io_pending_link);
 420                         io = NULL;
 421                 } else {
 422                         hio->eq = io->hw_priv;
 423                 }
 424         }
 425
 426         /* Must drop the lock before dispatching the IO */
 427         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 428
 429         if (!io)
 430                 return NULL;
 431
 432         /*
 433          * We pulled an IO off the pending list,
 434          * and either got an HW IO or don't need one
 435          */
 436         atomic_sub_return(1, &xport->io_pending_count);
 437         if (!hio)
 438                 status = efct_scsi_io_dispatch_no_hw_io(io);
 439         else
 440                 status = efct_scsi_io_dispatch_hw_io(io, hio);
 441         if (status) {
 442                 /*
 443                  * Invoke the HW callback, but do so in the
 444                  * separate execution context,provided by the
 445                  * NOP mailbox completion processing context
 446                  * by using efct_hw_async_call()
 447                  */
 448                 if (efct_hw_async_call(&efct->hw,
 449                                        efct_scsi_check_pending_async_cb, io)) {
 450                         efc_log_debug(efct, "call hw async failed\n");
 451                 }
 452         }
 453
 454         return io;
 455 }
 456
 457 void
 458 efct_scsi_check_pending(struct efct *efct)
 459 {
 460         struct efct_xport *xport = efct->xport;
 461         struct efct_io *io = NULL;
 462         int count = 0;
 463         unsigned long flags = 0;
 464         int dispatch = 0;
 465
 466         /* Guard against recursion */
 467         if (atomic_add_return(1, &xport->io_pending_recursing)) {
 468                 /* This function is already running.  Decrement and return. */
 469                 atomic_sub_return(1, &xport->io_pending_recursing);
 470                 return;
 471         }
 472
 473         while (efct_scsi_dispatch_pending(efct))
 474                 count++;
 475
 476         if (count) {
 477                 atomic_sub_return(1, &xport->io_pending_recursing);
 478                 return;
 479         }
 480
 481         /*
 482          * If nothing was removed from the list,
 483          * we might be in a case where we need to abort an
 484          * active IO and the abort is on the pending list.
 485          * Look for an abort we can dispatch.
 486          */
 487
 488         spin_lock_irqsave(&xport->io_pending_lock, flags);
 489
 490         list_for_each_entry(io, &xport->io_pending_list, io_pending_link) {
 491                 if (io->io_type == EFCT_IO_TYPE_ABORT && io->io_to_abort->hio) {
 492                         /* This IO has a HW IO, so it is
 493                          * active.  Dispatch the abort.
 494                          */
 495                         dispatch = 1;
 496                         list_del_init(&io->io_pending_link);
 497                         atomic_sub_return(1, &xport->io_pending_count);
 498                         break;
 499                 }
 500         }
 501
 502         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 503
 504         if (dispatch) {
 505                 if (efct_scsi_io_dispatch_no_hw_io(io)) {
 506                         if (efct_hw_async_call(&efct->hw,
 507                                 efct_scsi_check_pending_async_cb, io)) {
 508                                 efc_log_debug(efct, "hw async failed\n");
 509                         }
 510                 }
 511         }
 512
 513         atomic_sub_return(1, &xport->io_pending_recursing);
 514 }
 515
 516 int
 517 efct_scsi_io_dispatch(struct efct_io *io, void *cb)
 518 {
 519         struct efct_hw_io *hio;
 520         struct efct *efct = io->efct;
 521         struct efct_xport *xport = efct->xport;
 522         unsigned long flags = 0;
 523
 524         io->hw_cb = cb;
 525
 526         /*
 527          * if this IO already has a HW IO, then this is either
 528          * not the first phase of the IO. Send it to the HW.
 529          */
 530         if (io->hio)
 531                 return efct_scsi_io_dispatch_hw_io(io, io->hio);
 532
 533         /*
 534          * We don't already have a HW IO associated with the IO. First check
 535          * the pending list. If not empty, add IO to the tail and process the
 536          * pending list.
 537          */
 538         spin_lock_irqsave(&xport->io_pending_lock, flags);
 539         if (!list_empty(&xport->io_pending_list)) {
 540                 /*
 541                  * If this is a low latency request,
 542                  * the put at the front of the IO pending
 543                  * queue, otherwise put it at the end of the queue.
 544                  */
 545                 if (io->low_latency) {
 546                         INIT_LIST_HEAD(&io->io_pending_link);
 547                         list_add(&xport->io_pending_list, &io->io_pending_link);
 548                 } else {
 549                         INIT_LIST_HEAD(&io->io_pending_link);
 550                         list_add_tail(&io->io_pending_link,
 551                                       &xport->io_pending_list);
 552                 }
 553                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 554                 atomic_add_return(1, &xport->io_pending_count);
 555                 atomic_add_return(1, &xport->io_total_pending);
 556
 557                 /* process pending list */
 558                 efct_scsi_check_pending(efct);
 559                 return 0;
 560         }
 561         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 562
 563         /*
 564          * We don't have a HW IO associated with the IO and there's nothing
 565          * on the pending list. Attempt to allocate a HW IO and dispatch it.
 566          */
 567         hio = efct_hw_io_alloc(&io->efct->hw);
 568         if (!hio) {
 569                 /* Couldn't get a HW IO. Save this IO on the pending list */
 570                 spin_lock_irqsave(&xport->io_pending_lock, flags);
 571                 INIT_LIST_HEAD(&io->io_pending_link);
 572                 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 573                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 574
 575                 atomic_add_return(1, &xport->io_total_pending);
 576                 atomic_add_return(1, &xport->io_pending_count);
 577                 return 0;
 578         }
 579
 580         /* We successfully allocated a HW IO; dispatch to HW */
 581         return efct_scsi_io_dispatch_hw_io(io, hio);
 582 }
 583
 584 int
 585 efct_scsi_io_dispatch_abort(struct efct_io *io, void *cb)
 586 {
 587         struct efct *efct = io->efct;
 588         struct efct_xport *xport = efct->xport;
 589         unsigned long flags = 0;
 590
 591         io->hw_cb = cb;
 592
 593         /*
 594          * For aborts, we don't need a HW IO, but we still want
 595          * to pass through the pending list to preserve ordering.
 596          * Thus, if the pending list is not empty, add this abort
 597          * to the pending list and process the pending list.
 598          */
 599         spin_lock_irqsave(&xport->io_pending_lock, flags);
 600         if (!list_empty(&xport->io_pending_list)) {
 601                 INIT_LIST_HEAD(&io->io_pending_link);
 602                 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 603                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 604                 atomic_add_return(1, &xport->io_pending_count);
 605                 atomic_add_return(1, &xport->io_total_pending);
 606
 607                 /* process pending list */
 608                 efct_scsi_check_pending(efct);
 609                 return 0;
 610         }
 611         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 612
 613         /* nothing on pending list, dispatch abort */
 614         return efct_scsi_io_dispatch_no_hw_io(io);
 615 }
 616
 617 static inline int
 618 efct_scsi_xfer_data(struct efct_io *io, u32 flags,
 619                     struct efct_scsi_sgl *sgl, u32 sgl_count, u64 xwire_len,
 620                     enum efct_hw_io_type type, int enable_ar,
 621                     efct_scsi_io_cb_t cb, void *arg)
 622 {
 623         struct efct *efct;
 624         size_t residual = 0;
 625
 626         io->sgl_count = sgl_count;
 627
 628         efct = io->efct;
 629
 630         scsi_io_trace(io, "%s wire_len %llu\n",
 631                       (type == EFCT_HW_IO_TARGET_READ) ? "send" : "recv",
 632                       xwire_len);
 633
 634         io->hio_type = type;
 635
 636         io->scsi_tgt_cb = cb;
 637         io->scsi_tgt_cb_arg = arg;
 638
 639         residual = io->exp_xfer_len - io->transferred;
 640         io->wire_len = (xwire_len < residual) ? xwire_len : residual;
 641         residual = (xwire_len - io->wire_len);
 642
 643         memset(&io->iparam, 0, sizeof(io->iparam));
 644         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 645         io->iparam.fcp_tgt.offset = io->transferred;
 646         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 647         io->iparam.fcp_tgt.timeout = io->timeout;
 648
 649         /* if this is the last data phase and there is no residual, enable
 650          * auto-good-response
 651          */
 652         if (enable_ar && (flags & EFCT_SCSI_LAST_DATAPHASE) && residual == 0 &&
 653             ((io->transferred + io->wire_len) == io->exp_xfer_len) &&
 654             (!(flags & EFCT_SCSI_NO_AUTO_RESPONSE))) {
 655                 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 656                 io->auto_resp = true;
 657         } else {
 658                 io->auto_resp = false;
 659         }
 660
 661         /* save this transfer length */
 662         io->xfer_req = io->wire_len;
 663
 664         /* Adjust the transferred count to account for overrun
 665          * when the residual is calculated in efct_scsi_send_resp
 666          */
 667         io->transferred += residual;
 668
 669         /* Adjust the SGL size if there is overrun */
 670
 671         if (residual) {
 672                 struct efct_scsi_sgl  *sgl_ptr = &io->sgl[sgl_count - 1];
 673
 674                 while (residual) {
 675                         size_t len = sgl_ptr->len;
 676
 677                         if (len > residual) {
 678                                 sgl_ptr->len = len - residual;
 679                                 residual = 0;
 680                         } else {
 681                                 sgl_ptr->len = 0;
 682                                 residual -= len;
 683                                 io->sgl_count--;
 684                         }
 685                         sgl_ptr--;
 686                 }
 687         }
 688
 689         /* Set latency and WQ steering */
 690         io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 691         io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 692                                 EFCT_SCSI_WQ_STEERING_SHIFT;
 693         io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 694                                 EFCT_SCSI_WQ_CLASS_SHIFT;
 695
 696         if (efct->xport) {
 697                 struct efct_xport *xport = efct->xport;
 698
 699                 if (type == EFCT_HW_IO_TARGET_READ) {
 700                         xport->fcp_stats.input_requests++;
 701                         xport->fcp_stats.input_bytes += xwire_len;
 702                 } else if (type == EFCT_HW_IO_TARGET_WRITE) {
 703                         xport->fcp_stats.output_requests++;
 704                         xport->fcp_stats.output_bytes += xwire_len;
 705                 }
 706         }
 707         return efct_scsi_io_dispatch(io, efct_target_io_cb);
 708 }
 709
 710 int
 711 efct_scsi_send_rd_data(struct efct_io *io, u32 flags,
 712                        struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 713                        efct_scsi_io_cb_t cb, void *arg)
 714 {
 715         return efct_scsi_xfer_data(io, flags, sgl, sgl_count,
 716                                    len, EFCT_HW_IO_TARGET_READ,
 717                                    enable_tsend_auto_resp(io->efct), cb, arg);
 718 }
 719
 720 int
 721 efct_scsi_recv_wr_data(struct efct_io *io, u32 flags,
 722                        struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 723                        efct_scsi_io_cb_t cb, void *arg)
 724 {
 725         return efct_scsi_xfer_data(io, flags, sgl, sgl_count, len,
 726                                    EFCT_HW_IO_TARGET_WRITE,
 727                                    enable_treceive_auto_resp(io->efct), cb, arg);
 728 }
 729
 730 int
 731 efct_scsi_send_resp(struct efct_io *io, u32 flags,
 732                     struct efct_scsi_cmd_resp *rsp,
 733                     efct_scsi_io_cb_t cb, void *arg)
 734 {
 735         struct efct *efct;
 736         int residual;
 737         /* Always try auto resp */
 738         bool auto_resp = true;
 739         u8 scsi_status = 0;
 740         u16 scsi_status_qualifier = 0;
 741         u8 *sense_data = NULL;
 742         u32 sense_data_length = 0;
 743
 744         efct = io->efct;
 745
 746         if (rsp) {
 747                 scsi_status = rsp->scsi_status;
 748                 scsi_status_qualifier = rsp->scsi_status_qualifier;
 749                 sense_data = rsp->sense_data;
 750                 sense_data_length = rsp->sense_data_length;
 751                 residual = rsp->residual;
 752         } else {
 753                 residual = io->exp_xfer_len - io->transferred;
 754         }
 755
 756         io->wire_len = 0;
 757         io->hio_type = EFCT_HW_IO_TARGET_RSP;
 758
 759         io->scsi_tgt_cb = cb;
 760         io->scsi_tgt_cb_arg = arg;
 761
 762         memset(&io->iparam, 0, sizeof(io->iparam));
 763         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 764         io->iparam.fcp_tgt.offset = 0;
 765         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 766         io->iparam.fcp_tgt.timeout = io->timeout;
 767
 768         /* Set low latency queueing request */
 769         io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 770         io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 771                                 EFCT_SCSI_WQ_STEERING_SHIFT;
 772         io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 773                                 EFCT_SCSI_WQ_CLASS_SHIFT;
 774
 775         if (scsi_status != 0 || residual || sense_data_length) {
 776                 struct fcp_resp_with_ext *fcprsp = io->rspbuf.virt;
 777                 u8 *sns_data;
 778
 779                 if (!fcprsp) {
 780                         efc_log_err(efct, "NULL response buffer\n");
 781                         return -EIO;
 782                 }
 783
 784                 sns_data = (u8 *)io->rspbuf.virt + sizeof(*fcprsp);
 785
 786                 auto_resp = false;
 787
 788                 memset(fcprsp, 0, sizeof(*fcprsp));
 789
 790                 io->wire_len += sizeof(*fcprsp);
 791
 792                 fcprsp->resp.fr_status = scsi_status;
 793                 fcprsp->resp.fr_retry_delay =
 794                         cpu_to_be16(scsi_status_qualifier);
 795
 796                 /* set residual status if necessary */
 797                 if (residual != 0) {
 798                         /* FCP: if data transferred is less than the
 799                          * amount expected, then this is an underflow.
 800                          * If data transferred would have been greater
 801                          * than the amount expected this is an overflow
 802                          */
 803                         if (residual > 0) {
 804                                 fcprsp->resp.fr_flags |= FCP_RESID_UNDER;
 805                                 fcprsp->ext.fr_resid =  cpu_to_be32(residual);
 806                         } else {
 807                                 fcprsp->resp.fr_flags |= FCP_RESID_OVER;
 808                                 fcprsp->ext.fr_resid = cpu_to_be32(-residual);
 809                         }
 810                 }
 811
 812                 if (EFCT_SCSI_SNS_BUF_VALID(sense_data) && sense_data_length) {
 813                         if (sense_data_length > SCSI_SENSE_BUFFERSIZE) {
 814                                 efc_log_err(efct, "Sense exceeds max size.\n");
 815                                 return -EIO;
 816                         }
 817
 818                         fcprsp->resp.fr_flags |= FCP_SNS_LEN_VAL;
 819                         memcpy(sns_data, sense_data, sense_data_length);
 820                         fcprsp->ext.fr_sns_len = cpu_to_be32(sense_data_length);
 821                         io->wire_len += sense_data_length;
 822                 }
 823
 824                 io->sgl[0].addr = io->rspbuf.phys;
 825                 io->sgl[0].dif_addr = 0;
 826                 io->sgl[0].len = io->wire_len;
 827                 io->sgl_count = 1;
 828         }
 829
 830         if (auto_resp)
 831                 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 832
 833         return efct_scsi_io_dispatch(io, efct_target_io_cb);
 834 }
 835
 836 static int
 837 efct_target_bls_resp_cb(struct efct_hw_io *hio, u32 length, int status,
 838                         u32 ext_status, void *app)
 839 {
 840         struct efct_io *io = app;
 841         struct efct *efct;
 842         enum efct_scsi_io_status bls_status;
 843
 844         efct = io->efct;
 845
 846         /* BLS isn't really a "SCSI" concept, but use SCSI status */
 847         if (status) {
 848                 io_error_log(io, "s=%#x x=%#x\n", status, ext_status);
 849                 bls_status = EFCT_SCSI_STATUS_ERROR;
 850         } else {
 851                 bls_status = EFCT_SCSI_STATUS_GOOD;
 852         }
 853
 854         if (io->bls_cb) {
 855                 efct_scsi_io_cb_t bls_cb = io->bls_cb;
 856                 void *bls_cb_arg = io->bls_cb_arg;
 857
 858                 io->bls_cb = NULL;
 859                 io->bls_cb_arg = NULL;
 860
 861                 /* invoke callback */
 862                 bls_cb(io, bls_status, 0, bls_cb_arg);
 863         }
 864
 865         efct_scsi_check_pending(efct);
 866         return 0;
 867 }
 868
 869 static int
 870 efct_target_send_bls_resp(struct efct_io *io,
 871                           efct_scsi_io_cb_t cb, void *arg)
 872 {
 873         struct efct_node *node = io->node;
 874         struct sli_bls_params *bls = &io->iparam.bls;
 875         struct efct *efct = node->efct;
 876         struct fc_ba_acc *acc;
 877         int rc;
 878
 879         /* fill out IO structure with everything needed to send BA_ACC */
 880         memset(&io->iparam, 0, sizeof(io->iparam));
 881         bls->ox_id = io->init_task_tag;
 882         bls->rx_id = io->abort_rx_id;
 883         bls->vpi = io->node->vpi;
 884         bls->rpi = io->node->rpi;
 885         bls->s_id = U32_MAX;
 886         bls->d_id = io->node->node_fc_id;
 887         bls->rpi_registered = true;
 888
 889         acc = (void *)bls->payload;
 890         acc->ba_ox_id = cpu_to_be16(bls->ox_id);
 891         acc->ba_rx_id = cpu_to_be16(bls->rx_id);
 892         acc->ba_high_seq_cnt = cpu_to_be16(U16_MAX);
 893
 894         /* generic io fields have already been populated */
 895
 896         /* set type and BLS-specific fields */
 897         io->io_type = EFCT_IO_TYPE_BLS_RESP;
 898         io->display_name = "bls_rsp";
 899         io->hio_type = EFCT_HW_BLS_ACC;
 900         io->bls_cb = cb;
 901         io->bls_cb_arg = arg;
 902
 903         /* dispatch IO */
 904         rc = efct_hw_bls_send(efct, FC_RCTL_BA_ACC, bls,
 905                               efct_target_bls_resp_cb, io);
 906         return rc;
 907 }
 908
 909 static int efct_bls_send_rjt_cb(struct efct_hw_io *hio, u32 length, int status,
 910                                 u32 ext_status, void *app)
 911 {
 912         struct efct_io *io = app;
 913
 914         efct_scsi_io_free(io);
 915         return 0;
 916 }
 917
 918 struct efct_io *
 919 efct_bls_send_rjt(struct efct_io *io, struct fc_frame_header *hdr)
 920 {
 921         struct efct_node *node = io->node;
 922         struct sli_bls_params *bls = &io->iparam.bls;
 923         struct efct *efct = node->efct;
 924         struct fc_ba_rjt *acc;
 925         int rc;
 926
 927         /* fill out BLS Response-specific fields */
 928         io->io_type = EFCT_IO_TYPE_BLS_RESP;
 929         io->display_name = "ba_rjt";
 930         io->hio_type = EFCT_HW_BLS_RJT;
 931         io->init_task_tag = be16_to_cpu(hdr->fh_ox_id);
 932
 933         /* fill out iparam fields */
 934         memset(&io->iparam, 0, sizeof(io->iparam));
 935         bls->ox_id = be16_to_cpu(hdr->fh_ox_id);
 936         bls->rx_id = be16_to_cpu(hdr->fh_rx_id);
 937         bls->vpi = io->node->vpi;
 938         bls->rpi = io->node->rpi;
 939         bls->s_id = U32_MAX;
 940         bls->d_id = io->node->node_fc_id;
 941         bls->rpi_registered = true;
 942
 943         acc = (void *)bls->payload;
 944         acc->br_reason = ELS_RJT_UNAB;
 945         acc->br_explan = ELS_EXPL_NONE;
 946
 947         rc = efct_hw_bls_send(efct, FC_RCTL_BA_RJT, bls, efct_bls_send_rjt_cb,
 948                               io);
 949         if (rc) {
 950                 efc_log_err(efct, "efct_scsi_io_dispatch() failed: %d\n", rc);
 951                 efct_scsi_io_free(io);
 952                 io = NULL;
 953         }
 954         return io;
 955 }
 956
 957 int
 958 efct_scsi_send_tmf_resp(struct efct_io *io,
 959                         enum efct_scsi_tmf_resp rspcode,
 960                         u8 addl_rsp_info[3],
 961                         efct_scsi_io_cb_t cb, void *arg)
 962 {
 963         int rc;
 964         struct {
 965                 struct fcp_resp_with_ext rsp_ext;
 966                 struct fcp_resp_rsp_info info;
 967         } *fcprsp;
 968         u8 fcp_rspcode;
 969
 970         io->wire_len = 0;
 971
 972         switch (rspcode) {
 973         case EFCT_SCSI_TMF_FUNCTION_COMPLETE:
 974                 fcp_rspcode = FCP_TMF_CMPL;
 975                 break;
 976         case EFCT_SCSI_TMF_FUNCTION_SUCCEEDED:
 977         case EFCT_SCSI_TMF_FUNCTION_IO_NOT_FOUND:
 978                 fcp_rspcode = FCP_TMF_CMPL;
 979                 break;
 980         case EFCT_SCSI_TMF_FUNCTION_REJECTED:
 981                 fcp_rspcode = FCP_TMF_REJECTED;
 982                 break;
 983         case EFCT_SCSI_TMF_INCORRECT_LOGICAL_UNIT_NUMBER:
 984                 fcp_rspcode = FCP_TMF_INVALID_LUN;
 985                 break;
 986         case EFCT_SCSI_TMF_SERVICE_DELIVERY:
 987                 fcp_rspcode = FCP_TMF_FAILED;
 988                 break;
 989         default:
 990                 fcp_rspcode = FCP_TMF_REJECTED;
 991                 break;
 992         }
 993
 994         io->hio_type = EFCT_HW_IO_TARGET_RSP;
 995
 996         io->scsi_tgt_cb = cb;
 997         io->scsi_tgt_cb_arg = arg;
 998
 999         if (io->tmf_cmd == EFCT_SCSI_TMF_ABORT_TASK) {
1000                 rc = efct_target_send_bls_resp(io, cb, arg);
1001                 return rc;
1002         }
1003
1004         /* populate the FCP TMF response */
1005         fcprsp = io->rspbuf.virt;
1006         memset(fcprsp, 0, sizeof(*fcprsp));
1007
1008         fcprsp->rsp_ext.resp.fr_flags |= FCP_SNS_LEN_VAL;
1009
1010         if (addl_rsp_info) {
1011                 memcpy(fcprsp->info._fr_resvd, addl_rsp_info,
1012                        sizeof(fcprsp->info._fr_resvd));
1013         }
1014         fcprsp->info.rsp_code = fcp_rspcode;
1015
1016         io->wire_len = sizeof(*fcprsp);
1017
1018         fcprsp->rsp_ext.ext.fr_rsp_len =
1019                         cpu_to_be32(sizeof(struct fcp_resp_rsp_info));
1020
1021         io->sgl[0].addr = io->rspbuf.phys;
1022         io->sgl[0].dif_addr = 0;
1023         io->sgl[0].len = io->wire_len;
1024         io->sgl_count = 1;
1025
1026         memset(&io->iparam, 0, sizeof(io->iparam));
1027         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
1028         io->iparam.fcp_tgt.offset = 0;
1029         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
1030         io->iparam.fcp_tgt.timeout = io->timeout;
1031
1032         rc = efct_scsi_io_dispatch(io, efct_target_io_cb);
1033
1034         return rc;
1035 }
1036
1037 static int
1038 efct_target_abort_cb(struct efct_hw_io *hio, u32 length, int status,
1039                      u32 ext_status, void *app)
1040 {
1041         struct efct_io *io = app;
1042         struct efct *efct;
1043         enum efct_scsi_io_status scsi_status;
1044         efct_scsi_io_cb_t abort_cb;
1045         void *abort_cb_arg;
1046
1047         efct = io->efct;
1048
1049         if (!io->abort_cb)
1050                 goto done;
1051
1052         abort_cb = io->abort_cb;
1053         abort_cb_arg = io->abort_cb_arg;
1054
1055         io->abort_cb = NULL;
1056         io->abort_cb_arg = NULL;
1057
1058         switch (status) {
1059         case SLI4_FC_WCQE_STATUS_SUCCESS:
1060                 scsi_status = EFCT_SCSI_STATUS_GOOD;
1061                 break;
1062         case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
1063                 switch (ext_status) {
1064                 case SLI4_FC_LOCAL_REJECT_NO_XRI:
1065                         scsi_status = EFCT_SCSI_STATUS_NO_IO;
1066                         break;
1067                 case SLI4_FC_LOCAL_REJECT_ABORT_IN_PROGRESS:
1068                         scsi_status = EFCT_SCSI_STATUS_ABORT_IN_PROGRESS;
1069                         break;
1070                 default:
1071                         /*we have seen 0x15 (abort in progress)*/
1072                         scsi_status = EFCT_SCSI_STATUS_ERROR;
1073                         break;
1074                 }
1075                 break;
1076         case SLI4_FC_WCQE_STATUS_FCP_RSP_FAILURE:
1077                 scsi_status = EFCT_SCSI_STATUS_CHECK_RESPONSE;
1078                 break;
1079         default:
1080                 scsi_status = EFCT_SCSI_STATUS_ERROR;
1081                 break;
1082         }
1083         /* invoke callback */
1084         abort_cb(io->io_to_abort, scsi_status, 0, abort_cb_arg);
1085
1086 done:
1087         /* done with IO to abort,efct_ref_get(): efct_scsi_tgt_abort_io() */
1088         kref_put(&io->io_to_abort->ref, io->io_to_abort->release);
1089
1090         efct_io_pool_io_free(efct->xport->io_pool, io);
1091
1092         efct_scsi_check_pending(efct);
1093         return 0;
1094 }
1095
1096 int
1097 efct_scsi_tgt_abort_io(struct efct_io *io, efct_scsi_io_cb_t cb, void *arg)
1098 {
1099         struct efct *efct;
1100         struct efct_xport *xport;
1101         int rc;
1102         struct efct_io *abort_io = NULL;
1103
1104         efct = io->efct;
1105         xport = efct->xport;
1106
1107         /* take a reference on IO being aborted */
1108         if (kref_get_unless_zero(&io->ref) == 0) {
1109                 /* command no longer active */
1110                 scsi_io_printf(io, "command no longer active\n");
1111                 return -EIO;
1112         }
1113
1114         /*
1115          * allocate a new IO to send the abort request. Use efct_io_alloc()
1116          * directly, as we need an IO object that will not fail allocation
1117          * due to allocations being disabled (in efct_scsi_io_alloc())
1118          */
1119         abort_io = efct_io_pool_io_alloc(efct->xport->io_pool);
1120         if (!abort_io) {
1121                 atomic_add_return(1, &xport->io_alloc_failed_count);
1122                 kref_put(&io->ref, io->release);
1123                 return -EIO;
1124         }
1125
1126         /* Save the target server callback and argument */
1127         /* set generic fields */
1128         abort_io->cmd_tgt = true;
1129         abort_io->node = io->node;
1130
1131         /* set type and abort-specific fields */
1132         abort_io->io_type = EFCT_IO_TYPE_ABORT;
1133         abort_io->display_name = "tgt_abort";
1134         abort_io->io_to_abort = io;
1135         abort_io->send_abts = false;
1136         abort_io->abort_cb = cb;
1137         abort_io->abort_cb_arg = arg;
1138
1139         /* now dispatch IO */
1140         rc = efct_scsi_io_dispatch_abort(abort_io, efct_target_abort_cb);
1141         if (rc)
1142                 kref_put(&io->ref, io->release);
1143         return rc;
1144 }
1145
1146 void
1147 efct_scsi_io_complete(struct efct_io *io)
1148 {
1149         if (io->io_free) {
1150                 efc_log_debug(io->efct, "completion for non-busy io tag 0x%x\n",
1151                               io->tag);
1152                 return;
1153         }
1154
1155         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
1156         kref_put(&io->ref, io->release);
1157 }