GNU Linux-libre 4.14.259-gnu1
[releases.git] / drivers / scsi / qedf / qedf_io.c
1 /*
2  *  QLogic FCoE Offload Driver
3  *  Copyright (c) 2016-2017 Cavium Inc.
4  *
5  *  This software is available under the terms of the GNU General Public License
6  *  (GPL) Version 2, available from the file COPYING in the main directory of
7  *  this source tree.
8  */
9 #include <linux/spinlock.h>
10 #include <linux/vmalloc.h>
11 #include "qedf.h"
12 #include <scsi/scsi_tcq.h>
13
14 void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
15         unsigned int timer_msec)
16 {
17         queue_delayed_work(qedf->timer_work_queue, &io_req->timeout_work,
18             msecs_to_jiffies(timer_msec));
19 }
20
21 static void qedf_cmd_timeout(struct work_struct *work)
22 {
23
24         struct qedf_ioreq *io_req =
25             container_of(work, struct qedf_ioreq, timeout_work.work);
26         struct qedf_ctx *qedf = io_req->fcport->qedf;
27         struct qedf_rport *fcport = io_req->fcport;
28         u8 op = 0;
29
30         switch (io_req->cmd_type) {
31         case QEDF_ABTS:
32                 QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
33                     io_req->xid);
34                 /* Cleanup timed out ABTS */
35                 qedf_initiate_cleanup(io_req, true);
36                 complete(&io_req->abts_done);
37
38                 /*
39                  * Need to call kref_put for reference taken when initiate_abts
40                  * was called since abts_compl won't be called now that we've
41                  * cleaned up the task.
42                  */
43                 kref_put(&io_req->refcount, qedf_release_cmd);
44
45                 /*
46                  * Now that the original I/O and the ABTS are complete see
47                  * if we need to reconnect to the target.
48                  */
49                 qedf_restart_rport(fcport);
50                 break;
51         case QEDF_ELS:
52                 kref_get(&io_req->refcount);
53                 /*
54                  * Don't attempt to clean an ELS timeout as any subseqeunt
55                  * ABTS or cleanup requests just hang.  For now just free
56                  * the resources of the original I/O and the RRQ
57                  */
58                 QEDF_ERR(&(qedf->dbg_ctx), "ELS timeout, xid=0x%x.\n",
59                           io_req->xid);
60                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
61                 /* Call callback function to complete command */
62                 if (io_req->cb_func && io_req->cb_arg) {
63                         op = io_req->cb_arg->op;
64                         io_req->cb_func(io_req->cb_arg);
65                         io_req->cb_arg = NULL;
66                 }
67                 qedf_initiate_cleanup(io_req, true);
68                 kref_put(&io_req->refcount, qedf_release_cmd);
69                 break;
70         case QEDF_SEQ_CLEANUP:
71                 QEDF_ERR(&(qedf->dbg_ctx), "Sequence cleanup timeout, "
72                     "xid=0x%x.\n", io_req->xid);
73                 qedf_initiate_cleanup(io_req, true);
74                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
75                 qedf_process_seq_cleanup_compl(qedf, NULL, io_req);
76                 break;
77         default:
78                 break;
79         }
80 }
81
82 void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
83 {
84         struct io_bdt *bdt_info;
85         struct qedf_ctx *qedf = cmgr->qedf;
86         size_t bd_tbl_sz;
87         u16 min_xid = QEDF_MIN_XID;
88         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
89         int num_ios;
90         int i;
91         struct qedf_ioreq *io_req;
92
93         num_ios = max_xid - min_xid + 1;
94
95         /* Free fcoe_bdt_ctx structures */
96         if (!cmgr->io_bdt_pool)
97                 goto free_cmd_pool;
98
99         bd_tbl_sz = QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge);
100         for (i = 0; i < num_ios; i++) {
101                 bdt_info = cmgr->io_bdt_pool[i];
102                 if (bdt_info->bd_tbl) {
103                         dma_free_coherent(&qedf->pdev->dev, bd_tbl_sz,
104                             bdt_info->bd_tbl, bdt_info->bd_tbl_dma);
105                         bdt_info->bd_tbl = NULL;
106                 }
107         }
108
109         /* Destroy io_bdt pool */
110         for (i = 0; i < num_ios; i++) {
111                 kfree(cmgr->io_bdt_pool[i]);
112                 cmgr->io_bdt_pool[i] = NULL;
113         }
114
115         kfree(cmgr->io_bdt_pool);
116         cmgr->io_bdt_pool = NULL;
117
118 free_cmd_pool:
119
120         for (i = 0; i < num_ios; i++) {
121                 io_req = &cmgr->cmds[i];
122                 kfree(io_req->sgl_task_params);
123                 kfree(io_req->task_params);
124                 /* Make sure we free per command sense buffer */
125                 if (io_req->sense_buffer)
126                         dma_free_coherent(&qedf->pdev->dev,
127                             QEDF_SCSI_SENSE_BUFFERSIZE, io_req->sense_buffer,
128                             io_req->sense_buffer_dma);
129                 cancel_delayed_work_sync(&io_req->rrq_work);
130         }
131
132         /* Free command manager itself */
133         vfree(cmgr);
134 }
135
136 static void qedf_handle_rrq(struct work_struct *work)
137 {
138         struct qedf_ioreq *io_req =
139             container_of(work, struct qedf_ioreq, rrq_work.work);
140
141         qedf_send_rrq(io_req);
142
143 }
144
145 struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
146 {
147         struct qedf_cmd_mgr *cmgr;
148         struct io_bdt *bdt_info;
149         struct qedf_ioreq *io_req;
150         u16 xid;
151         int i;
152         int num_ios;
153         u16 min_xid = QEDF_MIN_XID;
154         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
155
156         /* Make sure num_queues is already set before calling this function */
157         if (!qedf->num_queues) {
158                 QEDF_ERR(&(qedf->dbg_ctx), "num_queues is not set.\n");
159                 return NULL;
160         }
161
162         if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN) {
163                 QEDF_WARN(&(qedf->dbg_ctx), "Invalid min_xid 0x%x and "
164                            "max_xid 0x%x.\n", min_xid, max_xid);
165                 return NULL;
166         }
167
168         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "min xid 0x%x, max xid "
169                    "0x%x.\n", min_xid, max_xid);
170
171         num_ios = max_xid - min_xid + 1;
172
173         cmgr = vzalloc(sizeof(struct qedf_cmd_mgr));
174         if (!cmgr) {
175                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc cmd mgr.\n");
176                 return NULL;
177         }
178
179         cmgr->qedf = qedf;
180         spin_lock_init(&cmgr->lock);
181
182         /*
183          * Initialize I/O request fields.
184          */
185         xid = QEDF_MIN_XID;
186
187         for (i = 0; i < num_ios; i++) {
188                 io_req = &cmgr->cmds[i];
189                 INIT_DELAYED_WORK(&io_req->timeout_work, qedf_cmd_timeout);
190
191                 io_req->xid = xid++;
192
193                 INIT_DELAYED_WORK(&io_req->rrq_work, qedf_handle_rrq);
194
195                 /* Allocate DMA memory to hold sense buffer */
196                 io_req->sense_buffer = dma_alloc_coherent(&qedf->pdev->dev,
197                     QEDF_SCSI_SENSE_BUFFERSIZE, &io_req->sense_buffer_dma,
198                     GFP_KERNEL);
199                 if (!io_req->sense_buffer)
200                         goto mem_err;
201
202                 /* Allocate task parameters to pass to f/w init funcions */
203                 io_req->task_params = kzalloc(sizeof(*io_req->task_params),
204                                               GFP_KERNEL);
205                 if (!io_req->task_params) {
206                         QEDF_ERR(&(qedf->dbg_ctx),
207                                  "Failed to allocate task_params for xid=0x%x\n",
208                                  i);
209                         goto mem_err;
210                 }
211
212                 /*
213                  * Allocate scatter/gather list info to pass to f/w init
214                  * functions.
215                  */
216                 io_req->sgl_task_params = kzalloc(
217                     sizeof(struct scsi_sgl_task_params), GFP_KERNEL);
218                 if (!io_req->sgl_task_params) {
219                         QEDF_ERR(&(qedf->dbg_ctx),
220                                  "Failed to allocate sgl_task_params for xid=0x%x\n",
221                                  i);
222                         goto mem_err;
223                 }
224         }
225
226         /* Allocate pool of io_bdts - one for each qedf_ioreq */
227         cmgr->io_bdt_pool = kmalloc_array(num_ios, sizeof(struct io_bdt *),
228             GFP_KERNEL);
229
230         if (!cmgr->io_bdt_pool) {
231                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc io_bdt_pool.\n");
232                 goto mem_err;
233         }
234
235         for (i = 0; i < num_ios; i++) {
236                 cmgr->io_bdt_pool[i] = kmalloc(sizeof(struct io_bdt),
237                     GFP_KERNEL);
238                 if (!cmgr->io_bdt_pool[i]) {
239                         QEDF_WARN(&(qedf->dbg_ctx),
240                                   "Failed to alloc io_bdt_pool[%d].\n", i);
241                         goto mem_err;
242                 }
243         }
244
245         for (i = 0; i < num_ios; i++) {
246                 bdt_info = cmgr->io_bdt_pool[i];
247                 bdt_info->bd_tbl = dma_alloc_coherent(&qedf->pdev->dev,
248                     QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge),
249                     &bdt_info->bd_tbl_dma, GFP_KERNEL);
250                 if (!bdt_info->bd_tbl) {
251                         QEDF_WARN(&(qedf->dbg_ctx),
252                                   "Failed to alloc bdt_tbl[%d].\n", i);
253                         goto mem_err;
254                 }
255         }
256         atomic_set(&cmgr->free_list_cnt, num_ios);
257         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
258             "cmgr->free_list_cnt=%d.\n",
259             atomic_read(&cmgr->free_list_cnt));
260
261         return cmgr;
262
263 mem_err:
264         qedf_cmd_mgr_free(cmgr);
265         return NULL;
266 }
267
268 struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
269 {
270         struct qedf_ctx *qedf = fcport->qedf;
271         struct qedf_cmd_mgr *cmd_mgr = qedf->cmd_mgr;
272         struct qedf_ioreq *io_req = NULL;
273         struct io_bdt *bd_tbl;
274         u16 xid;
275         uint32_t free_sqes;
276         int i;
277         unsigned long flags;
278
279         free_sqes = atomic_read(&fcport->free_sqes);
280
281         if (!free_sqes) {
282                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
283                     "Returning NULL, free_sqes=%d.\n ",
284                     free_sqes);
285                 goto out_failed;
286         }
287
288         /* Limit the number of outstanding R/W tasks */
289         if ((atomic_read(&fcport->num_active_ios) >=
290             NUM_RW_TASKS_PER_CONNECTION)) {
291                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
292                     "Returning NULL, num_active_ios=%d.\n",
293                     atomic_read(&fcport->num_active_ios));
294                 goto out_failed;
295         }
296
297         /* Limit global TIDs certain tasks */
298         if (atomic_read(&cmd_mgr->free_list_cnt) <= GBL_RSVD_TASKS) {
299                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
300                     "Returning NULL, free_list_cnt=%d.\n",
301                     atomic_read(&cmd_mgr->free_list_cnt));
302                 goto out_failed;
303         }
304
305         spin_lock_irqsave(&cmd_mgr->lock, flags);
306         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
307                 io_req = &cmd_mgr->cmds[cmd_mgr->idx];
308                 cmd_mgr->idx++;
309                 if (cmd_mgr->idx == FCOE_PARAMS_NUM_TASKS)
310                         cmd_mgr->idx = 0;
311
312                 /* Check to make sure command was previously freed */
313                 if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags))
314                         break;
315         }
316
317         if (i == FCOE_PARAMS_NUM_TASKS) {
318                 spin_unlock_irqrestore(&cmd_mgr->lock, flags);
319                 goto out_failed;
320         }
321
322         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
323         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
324
325         atomic_inc(&fcport->num_active_ios);
326         atomic_dec(&fcport->free_sqes);
327         xid = io_req->xid;
328         atomic_dec(&cmd_mgr->free_list_cnt);
329
330         io_req->cmd_mgr = cmd_mgr;
331         io_req->fcport = fcport;
332
333         /* Hold the io_req against deletion */
334         kref_init(&io_req->refcount);
335
336         /* Bind io_bdt for this io_req */
337         /* Have a static link between io_req and io_bdt_pool */
338         bd_tbl = io_req->bd_tbl = cmd_mgr->io_bdt_pool[xid];
339         if (bd_tbl == NULL) {
340                 QEDF_ERR(&(qedf->dbg_ctx), "bd_tbl is NULL, xid=%x.\n", xid);
341                 kref_put(&io_req->refcount, qedf_release_cmd);
342                 goto out_failed;
343         }
344         bd_tbl->io_req = io_req;
345         io_req->cmd_type = cmd_type;
346         io_req->tm_flags = 0;
347
348         /* Reset sequence offset data */
349         io_req->rx_buf_off = 0;
350         io_req->tx_buf_off = 0;
351         io_req->rx_id = 0xffff; /* No OX_ID */
352
353         return io_req;
354
355 out_failed:
356         /* Record failure for stats and return NULL to caller */
357         qedf->alloc_failures++;
358         return NULL;
359 }
360
361 static void qedf_free_mp_resc(struct qedf_ioreq *io_req)
362 {
363         struct qedf_mp_req *mp_req = &(io_req->mp_req);
364         struct qedf_ctx *qedf = io_req->fcport->qedf;
365         uint64_t sz = sizeof(struct scsi_sge);
366
367         /* clear tm flags */
368         if (mp_req->mp_req_bd) {
369                 dma_free_coherent(&qedf->pdev->dev, sz,
370                     mp_req->mp_req_bd, mp_req->mp_req_bd_dma);
371                 mp_req->mp_req_bd = NULL;
372         }
373         if (mp_req->mp_resp_bd) {
374                 dma_free_coherent(&qedf->pdev->dev, sz,
375                     mp_req->mp_resp_bd, mp_req->mp_resp_bd_dma);
376                 mp_req->mp_resp_bd = NULL;
377         }
378         if (mp_req->req_buf) {
379                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
380                     mp_req->req_buf, mp_req->req_buf_dma);
381                 mp_req->req_buf = NULL;
382         }
383         if (mp_req->resp_buf) {
384                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
385                     mp_req->resp_buf, mp_req->resp_buf_dma);
386                 mp_req->resp_buf = NULL;
387         }
388 }
389
390 void qedf_release_cmd(struct kref *ref)
391 {
392         struct qedf_ioreq *io_req =
393             container_of(ref, struct qedf_ioreq, refcount);
394         struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
395         struct qedf_rport *fcport = io_req->fcport;
396
397         if (io_req->cmd_type == QEDF_ELS ||
398             io_req->cmd_type == QEDF_TASK_MGMT_CMD)
399                 qedf_free_mp_resc(io_req);
400
401         atomic_inc(&cmd_mgr->free_list_cnt);
402         atomic_dec(&fcport->num_active_ios);
403         if (atomic_read(&fcport->num_active_ios) < 0)
404                 QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
405
406         /* Increment task retry identifier now that the request is released */
407         io_req->task_retry_identifier++;
408
409         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
410 }
411
412 static int qedf_split_bd(struct qedf_ioreq *io_req, u64 addr, int sg_len,
413         int bd_index)
414 {
415         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
416         int frag_size, sg_frags;
417
418         sg_frags = 0;
419         while (sg_len) {
420                 if (sg_len > QEDF_BD_SPLIT_SZ)
421                         frag_size = QEDF_BD_SPLIT_SZ;
422                 else
423                         frag_size = sg_len;
424                 bd[bd_index + sg_frags].sge_addr.lo = U64_LO(addr);
425                 bd[bd_index + sg_frags].sge_addr.hi = U64_HI(addr);
426                 bd[bd_index + sg_frags].sge_len = (uint16_t)frag_size;
427
428                 addr += (u64)frag_size;
429                 sg_frags++;
430                 sg_len -= frag_size;
431         }
432         return sg_frags;
433 }
434
435 static int qedf_map_sg(struct qedf_ioreq *io_req)
436 {
437         struct scsi_cmnd *sc = io_req->sc_cmd;
438         struct Scsi_Host *host = sc->device->host;
439         struct fc_lport *lport = shost_priv(host);
440         struct qedf_ctx *qedf = lport_priv(lport);
441         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
442         struct scatterlist *sg;
443         int byte_count = 0;
444         int sg_count = 0;
445         int bd_count = 0;
446         int sg_frags;
447         unsigned int sg_len;
448         u64 addr, end_addr;
449         int i;
450
451         sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
452             scsi_sg_count(sc), sc->sc_data_direction);
453
454         sg = scsi_sglist(sc);
455
456         /*
457          * New condition to send single SGE as cached-SGL with length less
458          * than 64k.
459          */
460         if ((sg_count == 1) && (sg_dma_len(sg) <=
461             QEDF_MAX_SGLEN_FOR_CACHESGL)) {
462                 sg_len = sg_dma_len(sg);
463                 addr = (u64)sg_dma_address(sg);
464
465                 bd[bd_count].sge_addr.lo = (addr & 0xffffffff);
466                 bd[bd_count].sge_addr.hi = (addr >> 32);
467                 bd[bd_count].sge_len = (u16)sg_len;
468
469                 return ++bd_count;
470         }
471
472         scsi_for_each_sg(sc, sg, sg_count, i) {
473                 sg_len = sg_dma_len(sg);
474                 addr = (u64)sg_dma_address(sg);
475                 end_addr = (u64)(addr + sg_len);
476
477                 /*
478                  * First s/g element in the list so check if the end_addr
479                  * is paged aligned. Also check to make sure the length is
480                  * at least page size.
481                  */
482                 if ((i == 0) && (sg_count > 1) &&
483                     ((end_addr % QEDF_PAGE_SIZE) ||
484                     sg_len < QEDF_PAGE_SIZE))
485                         io_req->use_slowpath = true;
486                 /*
487                  * Last s/g element so check if the start address is paged
488                  * aligned.
489                  */
490                 else if ((i == (sg_count - 1)) && (sg_count > 1) &&
491                     (addr % QEDF_PAGE_SIZE))
492                         io_req->use_slowpath = true;
493                 /*
494                  * Intermediate s/g element so check if start and end address
495                  * is page aligned.
496                  */
497                 else if ((i != 0) && (i != (sg_count - 1)) &&
498                     ((addr % QEDF_PAGE_SIZE) || (end_addr % QEDF_PAGE_SIZE)))
499                         io_req->use_slowpath = true;
500
501                 if (sg_len > QEDF_MAX_BD_LEN) {
502                         sg_frags = qedf_split_bd(io_req, addr, sg_len,
503                             bd_count);
504                 } else {
505                         sg_frags = 1;
506                         bd[bd_count].sge_addr.lo = U64_LO(addr);
507                         bd[bd_count].sge_addr.hi  = U64_HI(addr);
508                         bd[bd_count].sge_len = (uint16_t)sg_len;
509                 }
510
511                 bd_count += sg_frags;
512                 byte_count += sg_len;
513         }
514
515         if (byte_count != scsi_bufflen(sc))
516                 QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
517                           "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
518                            scsi_bufflen(sc), io_req->xid);
519
520         return bd_count;
521 }
522
523 static int qedf_build_bd_list_from_sg(struct qedf_ioreq *io_req)
524 {
525         struct scsi_cmnd *sc = io_req->sc_cmd;
526         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
527         int bd_count;
528
529         if (scsi_sg_count(sc)) {
530                 bd_count = qedf_map_sg(io_req);
531                 if (bd_count == 0)
532                         return -ENOMEM;
533         } else {
534                 bd_count = 0;
535                 bd[0].sge_addr.lo = bd[0].sge_addr.hi = 0;
536                 bd[0].sge_len = 0;
537         }
538         io_req->bd_tbl->bd_valid = bd_count;
539
540         return 0;
541 }
542
543 static void qedf_build_fcp_cmnd(struct qedf_ioreq *io_req,
544                                   struct fcp_cmnd *fcp_cmnd)
545 {
546         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
547
548         /* fcp_cmnd is 32 bytes */
549         memset(fcp_cmnd, 0, FCP_CMND_LEN);
550
551         /* 8 bytes: SCSI LUN info */
552         int_to_scsilun(sc_cmd->device->lun,
553                         (struct scsi_lun *)&fcp_cmnd->fc_lun);
554
555         /* 4 bytes: flag info */
556         fcp_cmnd->fc_pri_ta = 0;
557         fcp_cmnd->fc_tm_flags = io_req->tm_flags;
558         fcp_cmnd->fc_flags = io_req->io_req_flags;
559         fcp_cmnd->fc_cmdref = 0;
560
561         /* Populate data direction */
562         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
563                 fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
564         } else {
565                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
566                         fcp_cmnd->fc_flags |= FCP_CFL_WRDATA;
567                 else if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE)
568                         fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
569         }
570
571         fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
572
573         /* 16 bytes: CDB information */
574         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD)
575                 memcpy(fcp_cmnd->fc_cdb, sc_cmd->cmnd, sc_cmd->cmd_len);
576
577         /* 4 bytes: FCP data length */
578         fcp_cmnd->fc_dl = htonl(io_req->data_xfer_len);
579 }
580
581 static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
582         struct qedf_ioreq *io_req, struct fcoe_task_context *task_ctx,
583         struct fcoe_wqe *sqe)
584 {
585         enum fcoe_task_type task_type;
586         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
587         struct io_bdt *bd_tbl = io_req->bd_tbl;
588         u8 fcp_cmnd[32];
589         u32 tmp_fcp_cmnd[8];
590         int bd_count = 0;
591         struct qedf_ctx *qedf = fcport->qedf;
592         uint16_t cq_idx = smp_processor_id() % qedf->num_queues;
593         struct regpair sense_data_buffer_phys_addr;
594         u32 tx_io_size = 0;
595         u32 rx_io_size = 0;
596         int i, cnt;
597
598         /* Note init_initiator_rw_fcoe_task memsets the task context */
599         io_req->task = task_ctx;
600         memset(task_ctx, 0, sizeof(struct fcoe_task_context));
601         memset(io_req->task_params, 0, sizeof(struct fcoe_task_params));
602         memset(io_req->sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
603
604         /* Set task type bassed on DMA directio of command */
605         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
606                 task_type = FCOE_TASK_TYPE_READ_INITIATOR;
607         } else {
608                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
609                         task_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
610                         tx_io_size = io_req->data_xfer_len;
611                 } else {
612                         task_type = FCOE_TASK_TYPE_READ_INITIATOR;
613                         rx_io_size = io_req->data_xfer_len;
614                 }
615         }
616
617         /* Setup the fields for fcoe_task_params */
618         io_req->task_params->context = task_ctx;
619         io_req->task_params->sqe = sqe;
620         io_req->task_params->task_type = task_type;
621         io_req->task_params->tx_io_size = tx_io_size;
622         io_req->task_params->rx_io_size = rx_io_size;
623         io_req->task_params->conn_cid = fcport->fw_cid;
624         io_req->task_params->itid = io_req->xid;
625         io_req->task_params->cq_rss_number = cq_idx;
626         io_req->task_params->is_tape_device = fcport->dev_type;
627
628         /* Fill in information for scatter/gather list */
629         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD) {
630                 bd_count = bd_tbl->bd_valid;
631                 io_req->sgl_task_params->sgl = bd_tbl->bd_tbl;
632                 io_req->sgl_task_params->sgl_phys_addr.lo =
633                         U64_LO(bd_tbl->bd_tbl_dma);
634                 io_req->sgl_task_params->sgl_phys_addr.hi =
635                         U64_HI(bd_tbl->bd_tbl_dma);
636                 io_req->sgl_task_params->num_sges = bd_count;
637                 io_req->sgl_task_params->total_buffer_size =
638                     scsi_bufflen(io_req->sc_cmd);
639                 io_req->sgl_task_params->small_mid_sge =
640                         io_req->use_slowpath;
641         }
642
643         /* Fill in physical address of sense buffer */
644         sense_data_buffer_phys_addr.lo = U64_LO(io_req->sense_buffer_dma);
645         sense_data_buffer_phys_addr.hi = U64_HI(io_req->sense_buffer_dma);
646
647         /* fill FCP_CMND IU */
648         qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)tmp_fcp_cmnd);
649
650         /* Swap fcp_cmnd since FC is big endian */
651         cnt = sizeof(struct fcp_cmnd) / sizeof(u32);
652         for (i = 0; i < cnt; i++) {
653                 tmp_fcp_cmnd[i] = cpu_to_be32(tmp_fcp_cmnd[i]);
654         }
655         memcpy(fcp_cmnd, tmp_fcp_cmnd, sizeof(struct fcp_cmnd));
656
657         init_initiator_rw_fcoe_task(io_req->task_params,
658                                     io_req->sgl_task_params,
659                                     sense_data_buffer_phys_addr,
660                                     io_req->task_retry_identifier, fcp_cmnd);
661
662         /* Increment SGL type counters */
663         if (bd_count == 1) {
664                 qedf->single_sge_ios++;
665                 io_req->sge_type = QEDF_IOREQ_SINGLE_SGE;
666         } else if (io_req->use_slowpath) {
667                 qedf->slow_sge_ios++;
668                 io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
669         } else {
670                 qedf->fast_sge_ios++;
671                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
672         }
673 }
674
675 void qedf_init_mp_task(struct qedf_ioreq *io_req,
676         struct fcoe_task_context *task_ctx, struct fcoe_wqe *sqe)
677 {
678         struct qedf_mp_req *mp_req = &(io_req->mp_req);
679         struct qedf_rport *fcport = io_req->fcport;
680         struct qedf_ctx *qedf = io_req->fcport->qedf;
681         struct fc_frame_header *fc_hdr;
682         struct fcoe_tx_mid_path_params task_fc_hdr;
683         struct scsi_sgl_task_params tx_sgl_task_params;
684         struct scsi_sgl_task_params rx_sgl_task_params;
685
686         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
687                   "Initializing MP task for cmd_type=%d\n",
688                   io_req->cmd_type);
689
690         qedf->control_requests++;
691
692         memset(&tx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
693         memset(&rx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
694         memset(task_ctx, 0, sizeof(struct fcoe_task_context));
695         memset(&task_fc_hdr, 0, sizeof(struct fcoe_tx_mid_path_params));
696
697         /* Setup the task from io_req for easy reference */
698         io_req->task = task_ctx;
699
700         /* Setup the fields for fcoe_task_params */
701         io_req->task_params->context = task_ctx;
702         io_req->task_params->sqe = sqe;
703         io_req->task_params->task_type = FCOE_TASK_TYPE_MIDPATH;
704         io_req->task_params->tx_io_size = io_req->data_xfer_len;
705         /* rx_io_size tells the f/w how large a response buffer we have */
706         io_req->task_params->rx_io_size = PAGE_SIZE;
707         io_req->task_params->conn_cid = fcport->fw_cid;
708         io_req->task_params->itid = io_req->xid;
709         /* Return middle path commands on CQ 0 */
710         io_req->task_params->cq_rss_number = 0;
711         io_req->task_params->is_tape_device = fcport->dev_type;
712
713         fc_hdr = &(mp_req->req_fc_hdr);
714         /* Set OX_ID and RX_ID based on driver task id */
715         fc_hdr->fh_ox_id = io_req->xid;
716         fc_hdr->fh_rx_id = htons(0xffff);
717
718         /* Set up FC header information */
719         task_fc_hdr.parameter = fc_hdr->fh_parm_offset;
720         task_fc_hdr.r_ctl = fc_hdr->fh_r_ctl;
721         task_fc_hdr.type = fc_hdr->fh_type;
722         task_fc_hdr.cs_ctl = fc_hdr->fh_cs_ctl;
723         task_fc_hdr.df_ctl = fc_hdr->fh_df_ctl;
724         task_fc_hdr.rx_id = fc_hdr->fh_rx_id;
725         task_fc_hdr.ox_id = fc_hdr->fh_ox_id;
726
727         /* Set up s/g list parameters for request buffer */
728         tx_sgl_task_params.sgl = mp_req->mp_req_bd;
729         tx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_req_bd_dma);
730         tx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_req_bd_dma);
731         tx_sgl_task_params.num_sges = 1;
732         /* Set PAGE_SIZE for now since sg element is that size ??? */
733         tx_sgl_task_params.total_buffer_size = io_req->data_xfer_len;
734         tx_sgl_task_params.small_mid_sge = 0;
735
736         /* Set up s/g list parameters for request buffer */
737         rx_sgl_task_params.sgl = mp_req->mp_resp_bd;
738         rx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_resp_bd_dma);
739         rx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_resp_bd_dma);
740         rx_sgl_task_params.num_sges = 1;
741         /* Set PAGE_SIZE for now since sg element is that size ??? */
742         rx_sgl_task_params.total_buffer_size = PAGE_SIZE;
743         rx_sgl_task_params.small_mid_sge = 0;
744
745
746         /*
747          * Last arg is 0 as previous code did not set that we wanted the
748          * fc header information.
749          */
750         init_initiator_midpath_unsolicited_fcoe_task(io_req->task_params,
751                                                      &task_fc_hdr,
752                                                      &tx_sgl_task_params,
753                                                      &rx_sgl_task_params, 0);
754
755         /* Midpath requests always consume 1 SGE */
756         qedf->single_sge_ios++;
757 }
758
759 /* Presumed that fcport->rport_lock is held */
760 u16 qedf_get_sqe_idx(struct qedf_rport *fcport)
761 {
762         uint16_t total_sqe = (fcport->sq_mem_size)/(sizeof(struct fcoe_wqe));
763         u16 rval;
764
765         rval = fcport->sq_prod_idx;
766
767         /* Adjust ring index */
768         fcport->sq_prod_idx++;
769         fcport->fw_sq_prod_idx++;
770         if (fcport->sq_prod_idx == total_sqe)
771                 fcport->sq_prod_idx = 0;
772
773         return rval;
774 }
775
776 void qedf_ring_doorbell(struct qedf_rport *fcport)
777 {
778         struct fcoe_db_data dbell = { 0 };
779
780         dbell.agg_flags = 0;
781
782         dbell.params |= DB_DEST_XCM << FCOE_DB_DATA_DEST_SHIFT;
783         dbell.params |= DB_AGG_CMD_SET << FCOE_DB_DATA_AGG_CMD_SHIFT;
784         dbell.params |= DQ_XCM_FCOE_SQ_PROD_CMD <<
785             FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
786
787         dbell.sq_prod = fcport->fw_sq_prod_idx;
788         writel(*(u32 *)&dbell, fcport->p_doorbell);
789         /* Make sure SQ index is updated so f/w prcesses requests in order */
790         wmb();
791         mmiowb();
792 }
793
794 static void qedf_trace_io(struct qedf_rport *fcport, struct qedf_ioreq *io_req,
795                           int8_t direction)
796 {
797         struct qedf_ctx *qedf = fcport->qedf;
798         struct qedf_io_log *io_log;
799         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
800         unsigned long flags;
801         uint8_t op;
802
803         spin_lock_irqsave(&qedf->io_trace_lock, flags);
804
805         io_log = &qedf->io_trace_buf[qedf->io_trace_idx];
806         io_log->direction = direction;
807         io_log->task_id = io_req->xid;
808         io_log->port_id = fcport->rdata->ids.port_id;
809         io_log->lun = sc_cmd->device->lun;
810         io_log->op = op = sc_cmd->cmnd[0];
811         io_log->lba[0] = sc_cmd->cmnd[2];
812         io_log->lba[1] = sc_cmd->cmnd[3];
813         io_log->lba[2] = sc_cmd->cmnd[4];
814         io_log->lba[3] = sc_cmd->cmnd[5];
815         io_log->bufflen = scsi_bufflen(sc_cmd);
816         io_log->sg_count = scsi_sg_count(sc_cmd);
817         io_log->result = sc_cmd->result;
818         io_log->jiffies = jiffies;
819         io_log->refcount = kref_read(&io_req->refcount);
820
821         if (direction == QEDF_IO_TRACE_REQ) {
822                 /* For requests we only care abot the submission CPU */
823                 io_log->req_cpu = io_req->cpu;
824                 io_log->int_cpu = 0;
825                 io_log->rsp_cpu = 0;
826         } else if (direction == QEDF_IO_TRACE_RSP) {
827                 io_log->req_cpu = io_req->cpu;
828                 io_log->int_cpu = io_req->int_cpu;
829                 io_log->rsp_cpu = smp_processor_id();
830         }
831
832         io_log->sge_type = io_req->sge_type;
833
834         qedf->io_trace_idx++;
835         if (qedf->io_trace_idx == QEDF_IO_TRACE_SIZE)
836                 qedf->io_trace_idx = 0;
837
838         spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
839 }
840
841 int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
842 {
843         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
844         struct Scsi_Host *host = sc_cmd->device->host;
845         struct fc_lport *lport = shost_priv(host);
846         struct qedf_ctx *qedf = lport_priv(lport);
847         struct fcoe_task_context *task_ctx;
848         u16 xid;
849         enum fcoe_task_type req_type = 0;
850         struct fcoe_wqe *sqe;
851         u16 sqe_idx;
852
853         /* Initialize rest of io_req fileds */
854         io_req->data_xfer_len = scsi_bufflen(sc_cmd);
855         sc_cmd->SCp.ptr = (char *)io_req;
856         io_req->use_slowpath = false; /* Assume fast SGL by default */
857
858         /* Record which cpu this request is associated with */
859         io_req->cpu = smp_processor_id();
860
861         if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
862                 req_type = FCOE_TASK_TYPE_READ_INITIATOR;
863                 io_req->io_req_flags = QEDF_READ;
864                 qedf->input_requests++;
865         } else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
866                 req_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
867                 io_req->io_req_flags = QEDF_WRITE;
868                 qedf->output_requests++;
869         } else {
870                 io_req->io_req_flags = 0;
871                 qedf->control_requests++;
872         }
873
874         xid = io_req->xid;
875
876         /* Build buffer descriptor list for firmware from sg list */
877         if (qedf_build_bd_list_from_sg(io_req)) {
878                 QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
879                 kref_put(&io_req->refcount, qedf_release_cmd);
880                 return -EAGAIN;
881         }
882
883         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
884                 QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
885                 kref_put(&io_req->refcount, qedf_release_cmd);
886                 return -EINVAL;
887         }
888
889         /* Obtain free SQE */
890         sqe_idx = qedf_get_sqe_idx(fcport);
891         sqe = &fcport->sq[sqe_idx];
892         memset(sqe, 0, sizeof(struct fcoe_wqe));
893
894         /* Get the task context */
895         task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
896         if (!task_ctx) {
897                 QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
898                            xid);
899                 kref_put(&io_req->refcount, qedf_release_cmd);
900                 return -EINVAL;
901         }
902
903         qedf_init_task(fcport, lport, io_req, task_ctx, sqe);
904
905         /* Ring doorbell */
906         qedf_ring_doorbell(fcport);
907
908         if (qedf_io_tracing && io_req->sc_cmd)
909                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
910
911         return false;
912 }
913
914 int
915 qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
916 {
917         struct fc_lport *lport = shost_priv(host);
918         struct qedf_ctx *qedf = lport_priv(lport);
919         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
920         struct fc_rport_libfc_priv *rp = rport->dd_data;
921         struct qedf_rport *fcport = rport->dd_data;
922         struct qedf_ioreq *io_req;
923         int rc = 0;
924         int rval;
925         unsigned long flags = 0;
926
927
928         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
929             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
930                 sc_cmd->result = DID_NO_CONNECT << 16;
931                 sc_cmd->scsi_done(sc_cmd);
932                 return 0;
933         }
934
935         rval = fc_remote_port_chkready(rport);
936         if (rval) {
937                 sc_cmd->result = rval;
938                 sc_cmd->scsi_done(sc_cmd);
939                 return 0;
940         }
941
942         /* Retry command if we are doing a qed drain operation */
943         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
944                 rc = SCSI_MLQUEUE_HOST_BUSY;
945                 goto exit_qcmd;
946         }
947
948         if (lport->state != LPORT_ST_READY ||
949             atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
950                 rc = SCSI_MLQUEUE_HOST_BUSY;
951                 goto exit_qcmd;
952         }
953
954         /* rport and tgt are allocated together, so tgt should be non-NULL */
955         fcport = (struct qedf_rport *)&rp[1];
956
957         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
958                 /*
959                  * Session is not offloaded yet. Let SCSI-ml retry
960                  * the command.
961                  */
962                 rc = SCSI_MLQUEUE_TARGET_BUSY;
963                 goto exit_qcmd;
964         }
965         if (fcport->retry_delay_timestamp) {
966                 if (time_after(jiffies, fcport->retry_delay_timestamp)) {
967                         fcport->retry_delay_timestamp = 0;
968                 } else {
969                         /* If retry_delay timer is active, flow off the ML */
970                         rc = SCSI_MLQUEUE_TARGET_BUSY;
971                         goto exit_qcmd;
972                 }
973         }
974
975         io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
976         if (!io_req) {
977                 rc = SCSI_MLQUEUE_HOST_BUSY;
978                 goto exit_qcmd;
979         }
980
981         io_req->sc_cmd = sc_cmd;
982
983         /* Take fcport->rport_lock for posting to fcport send queue */
984         spin_lock_irqsave(&fcport->rport_lock, flags);
985         if (qedf_post_io_req(fcport, io_req)) {
986                 QEDF_WARN(&(qedf->dbg_ctx), "Unable to post io_req\n");
987                 /* Return SQE to pool */
988                 atomic_inc(&fcport->free_sqes);
989                 rc = SCSI_MLQUEUE_HOST_BUSY;
990         }
991         spin_unlock_irqrestore(&fcport->rport_lock, flags);
992
993 exit_qcmd:
994         return rc;
995 }
996
997 static void qedf_parse_fcp_rsp(struct qedf_ioreq *io_req,
998                                  struct fcoe_cqe_rsp_info *fcp_rsp)
999 {
1000         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
1001         struct qedf_ctx *qedf = io_req->fcport->qedf;
1002         u8 rsp_flags = fcp_rsp->rsp_flags.flags;
1003         int fcp_sns_len = 0;
1004         int fcp_rsp_len = 0;
1005         uint8_t *rsp_info, *sense_data;
1006
1007         io_req->fcp_status = FC_GOOD;
1008         io_req->fcp_resid = 0;
1009         if (rsp_flags & (FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER |
1010             FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER))
1011                 io_req->fcp_resid = fcp_rsp->fcp_resid;
1012
1013         io_req->scsi_comp_flags = rsp_flags;
1014         CMD_SCSI_STATUS(sc_cmd) = io_req->cdb_status =
1015             fcp_rsp->scsi_status_code;
1016
1017         if (rsp_flags &
1018             FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID)
1019                 fcp_rsp_len = fcp_rsp->fcp_rsp_len;
1020
1021         if (rsp_flags &
1022             FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID)
1023                 fcp_sns_len = fcp_rsp->fcp_sns_len;
1024
1025         io_req->fcp_rsp_len = fcp_rsp_len;
1026         io_req->fcp_sns_len = fcp_sns_len;
1027         rsp_info = sense_data = io_req->sense_buffer;
1028
1029         /* fetch fcp_rsp_code */
1030         if ((fcp_rsp_len == 4) || (fcp_rsp_len == 8)) {
1031                 /* Only for task management function */
1032                 io_req->fcp_rsp_code = rsp_info[3];
1033                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1034                     "fcp_rsp_code = %d\n", io_req->fcp_rsp_code);
1035                 /* Adjust sense-data location. */
1036                 sense_data += fcp_rsp_len;
1037         }
1038
1039         if (fcp_sns_len > SCSI_SENSE_BUFFERSIZE) {
1040                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1041                     "Truncating sense buffer\n");
1042                 fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
1043         }
1044
1045         /* The sense buffer can be NULL for TMF commands */
1046         if (sc_cmd->sense_buffer) {
1047                 memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
1048                 if (fcp_sns_len)
1049                         memcpy(sc_cmd->sense_buffer, sense_data,
1050                             fcp_sns_len);
1051         }
1052 }
1053
1054 static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
1055 {
1056         struct scsi_cmnd *sc = io_req->sc_cmd;
1057
1058         if (io_req->bd_tbl->bd_valid && sc && scsi_sg_count(sc)) {
1059                 dma_unmap_sg(&qedf->pdev->dev, scsi_sglist(sc),
1060                     scsi_sg_count(sc), sc->sc_data_direction);
1061                 io_req->bd_tbl->bd_valid = 0;
1062         }
1063 }
1064
1065 void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1066         struct qedf_ioreq *io_req)
1067 {
1068         u16 xid, rval;
1069         struct fcoe_task_context *task_ctx;
1070         struct scsi_cmnd *sc_cmd;
1071         struct fcoe_cqe_rsp_info *fcp_rsp;
1072         struct qedf_rport *fcport;
1073         int refcount;
1074         u16 scope, qualifier = 0;
1075         u8 fw_residual_flag = 0;
1076
1077         if (!io_req)
1078                 return;
1079         if (!cqe)
1080                 return;
1081
1082         xid = io_req->xid;
1083         task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
1084         sc_cmd = io_req->sc_cmd;
1085         fcp_rsp = &cqe->cqe_info.rsp_info;
1086
1087         if (!sc_cmd) {
1088                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1089                 return;
1090         }
1091
1092         if (!sc_cmd->SCp.ptr) {
1093                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1094                     "another context.\n");
1095                 return;
1096         }
1097
1098         if (!sc_cmd->request) {
1099                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
1100                     "sc_cmd=%p.\n", sc_cmd);
1101                 return;
1102         }
1103
1104         if (!sc_cmd->request->special) {
1105                 QEDF_WARN(&(qedf->dbg_ctx), "request->special is NULL so "
1106                     "request not valid, sc_cmd=%p.\n", sc_cmd);
1107                 return;
1108         }
1109
1110         if (!sc_cmd->request->q) {
1111                 QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request "
1112                    "is not valid, sc_cmd=%p.\n", sc_cmd);
1113                 return;
1114         }
1115
1116         fcport = io_req->fcport;
1117
1118         qedf_parse_fcp_rsp(io_req, fcp_rsp);
1119
1120         qedf_unmap_sg_list(qedf, io_req);
1121
1122         /* Check for FCP transport error */
1123         if (io_req->fcp_rsp_len > 3 && io_req->fcp_rsp_code) {
1124                 QEDF_ERR(&(qedf->dbg_ctx),
1125                     "FCP I/O protocol failure xid=0x%x fcp_rsp_len=%d "
1126                     "fcp_rsp_code=%d.\n", io_req->xid, io_req->fcp_rsp_len,
1127                     io_req->fcp_rsp_code);
1128                 sc_cmd->result = DID_BUS_BUSY << 16;
1129                 goto out;
1130         }
1131
1132         fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
1133             FCOE_CQE_RSP_INFO_FW_UNDERRUN);
1134         if (fw_residual_flag) {
1135                 QEDF_ERR(&(qedf->dbg_ctx),
1136                     "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x "
1137                     "fcp_resid=%d fw_residual=0x%x.\n", io_req->xid,
1138                     fcp_rsp->rsp_flags.flags, io_req->fcp_resid,
1139                     cqe->cqe_info.rsp_info.fw_residual);
1140
1141                 if (io_req->cdb_status == 0)
1142                         sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
1143                 else
1144                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1145
1146                 /* Abort the command since we did not get all the data */
1147                 init_completion(&io_req->abts_done);
1148                 rval = qedf_initiate_abts(io_req, true);
1149                 if (rval) {
1150                         QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1151                         sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
1152                 }
1153
1154                 /*
1155                  * Set resid to the whole buffer length so we won't try to resue
1156                  * any previously data.
1157                  */
1158                 scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1159                 goto out;
1160         }
1161
1162         switch (io_req->fcp_status) {
1163         case FC_GOOD:
1164                 if (io_req->cdb_status == 0) {
1165                         /* Good I/O completion */
1166                         sc_cmd->result = DID_OK << 16;
1167                 } else {
1168                         refcount = kref_read(&io_req->refcount);
1169                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1170                             "%d:0:%d:%lld xid=0x%0x op=0x%02x "
1171                             "lba=%02x%02x%02x%02x cdb_status=%d "
1172                             "fcp_resid=0x%x refcount=%d.\n",
1173                             qedf->lport->host->host_no, sc_cmd->device->id,
1174                             sc_cmd->device->lun, io_req->xid,
1175                             sc_cmd->cmnd[0], sc_cmd->cmnd[2], sc_cmd->cmnd[3],
1176                             sc_cmd->cmnd[4], sc_cmd->cmnd[5],
1177                             io_req->cdb_status, io_req->fcp_resid,
1178                             refcount);
1179                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1180
1181                         if (io_req->cdb_status == SAM_STAT_TASK_SET_FULL ||
1182                             io_req->cdb_status == SAM_STAT_BUSY) {
1183                                 /*
1184                                  * Check whether we need to set retry_delay at
1185                                  * all based on retry_delay module parameter
1186                                  * and the status qualifier.
1187                                  */
1188
1189                                 /* Upper 2 bits */
1190                                 scope = fcp_rsp->retry_delay_timer & 0xC000;
1191                                 /* Lower 14 bits */
1192                                 qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
1193
1194                                 if (qedf_retry_delay &&
1195                                     scope > 0 && qualifier > 0 &&
1196                                     qualifier <= 0x3FEF) {
1197                                         /* Check we don't go over the max */
1198                                         if (qualifier > QEDF_RETRY_DELAY_MAX)
1199                                                 qualifier =
1200                                                     QEDF_RETRY_DELAY_MAX;
1201                                         fcport->retry_delay_timestamp =
1202                                             jiffies + (qualifier * HZ / 10);
1203                                 }
1204                         }
1205                 }
1206                 if (io_req->fcp_resid)
1207                         scsi_set_resid(sc_cmd, io_req->fcp_resid);
1208                 break;
1209         default:
1210                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
1211                            io_req->fcp_status);
1212                 break;
1213         }
1214
1215 out:
1216         if (qedf_io_tracing)
1217                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
1218
1219         io_req->sc_cmd = NULL;
1220         sc_cmd->SCp.ptr =  NULL;
1221         sc_cmd->scsi_done(sc_cmd);
1222         kref_put(&io_req->refcount, qedf_release_cmd);
1223 }
1224
1225 /* Return a SCSI command in some other context besides a normal completion */
1226 void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
1227         int result)
1228 {
1229         u16 xid;
1230         struct scsi_cmnd *sc_cmd;
1231         int refcount;
1232
1233         if (!io_req)
1234                 return;
1235
1236         xid = io_req->xid;
1237         sc_cmd = io_req->sc_cmd;
1238
1239         if (!sc_cmd) {
1240                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1241                 return;
1242         }
1243
1244         if (!sc_cmd->SCp.ptr) {
1245                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1246                     "another context.\n");
1247                 return;
1248         }
1249
1250         qedf_unmap_sg_list(qedf, io_req);
1251
1252         sc_cmd->result = result << 16;
1253         refcount = kref_read(&io_req->refcount);
1254         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "%d:0:%d:%lld: Completing "
1255             "sc_cmd=%p result=0x%08x op=0x%02x lba=0x%02x%02x%02x%02x, "
1256             "allowed=%d retries=%d refcount=%d.\n",
1257             qedf->lport->host->host_no, sc_cmd->device->id,
1258             sc_cmd->device->lun, sc_cmd, sc_cmd->result, sc_cmd->cmnd[0],
1259             sc_cmd->cmnd[2], sc_cmd->cmnd[3], sc_cmd->cmnd[4],
1260             sc_cmd->cmnd[5], sc_cmd->allowed, sc_cmd->retries,
1261             refcount);
1262
1263         /*
1264          * Set resid to the whole buffer length so we won't try to resue any
1265          * previously read data
1266          */
1267         scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1268
1269         if (qedf_io_tracing)
1270                 qedf_trace_io(io_req->fcport, io_req, QEDF_IO_TRACE_RSP);
1271
1272         io_req->sc_cmd = NULL;
1273         sc_cmd->SCp.ptr = NULL;
1274         sc_cmd->scsi_done(sc_cmd);
1275         kref_put(&io_req->refcount, qedf_release_cmd);
1276 }
1277
1278 /*
1279  * Handle warning type CQE completions. This is mainly used for REC timer
1280  * popping.
1281  */
1282 void qedf_process_warning_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1283         struct qedf_ioreq *io_req)
1284 {
1285         int rval, i;
1286         struct qedf_rport *fcport = io_req->fcport;
1287         u64 err_warn_bit_map;
1288         u8 err_warn = 0xff;
1289
1290         if (!cqe)
1291                 return;
1292
1293         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Warning CQE, "
1294                   "xid=0x%x\n", io_req->xid);
1295         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1296                   "err_warn_bitmap=%08x:%08x\n",
1297                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1298                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1299         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1300                   "rx_buff_off=%08x, rx_id=%04x\n",
1301                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1302                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1303                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1304
1305         /* Normalize the error bitmap value to an just an unsigned int */
1306         err_warn_bit_map = (u64)
1307             ((u64)cqe->cqe_info.err_info.err_warn_bitmap_hi << 32) |
1308             (u64)cqe->cqe_info.err_info.err_warn_bitmap_lo;
1309         for (i = 0; i < 64; i++) {
1310                 if (err_warn_bit_map & (u64)((u64)1 << i)) {
1311                         err_warn = i;
1312                         break;
1313                 }
1314         }
1315
1316         /* Check if REC TOV expired if this is a tape device */
1317         if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
1318                 if (err_warn ==
1319                     FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION) {
1320                         QEDF_ERR(&(qedf->dbg_ctx), "REC timer expired.\n");
1321                         if (!test_bit(QEDF_CMD_SRR_SENT, &io_req->flags)) {
1322                                 io_req->rx_buf_off =
1323                                     cqe->cqe_info.err_info.rx_buf_off;
1324                                 io_req->tx_buf_off =
1325                                     cqe->cqe_info.err_info.tx_buf_off;
1326                                 io_req->rx_id = cqe->cqe_info.err_info.rx_id;
1327                                 rval = qedf_send_rec(io_req);
1328                                 /*
1329                                  * We only want to abort the io_req if we
1330                                  * can't queue the REC command as we want to
1331                                  * keep the exchange open for recovery.
1332                                  */
1333                                 if (rval)
1334                                         goto send_abort;
1335                         }
1336                         return;
1337                 }
1338         }
1339
1340 send_abort:
1341         init_completion(&io_req->abts_done);
1342         rval = qedf_initiate_abts(io_req, true);
1343         if (rval)
1344                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1345 }
1346
1347 /* Cleanup a command when we receive an error detection completion */
1348 void qedf_process_error_detect(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1349         struct qedf_ioreq *io_req)
1350 {
1351         int rval;
1352
1353         if (!cqe)
1354                 return;
1355
1356         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Error detection CQE, "
1357                   "xid=0x%x\n", io_req->xid);
1358         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1359                   "err_warn_bitmap=%08x:%08x\n",
1360                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1361                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1362         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1363                   "rx_buff_off=%08x, rx_id=%04x\n",
1364                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1365                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1366                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1367
1368         if (qedf->stop_io_on_error) {
1369                 qedf_stop_all_io(qedf);
1370                 return;
1371         }
1372
1373         init_completion(&io_req->abts_done);
1374         rval = qedf_initiate_abts(io_req, true);
1375         if (rval)
1376                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1377 }
1378
1379 static void qedf_flush_els_req(struct qedf_ctx *qedf,
1380         struct qedf_ioreq *els_req)
1381 {
1382         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1383             "Flushing ELS request xid=0x%x refcount=%d.\n", els_req->xid,
1384             kref_read(&els_req->refcount));
1385
1386         /*
1387          * Need to distinguish this from a timeout when calling the
1388          * els_req->cb_func.
1389          */
1390         els_req->event = QEDF_IOREQ_EV_ELS_FLUSH;
1391
1392         /* Cancel the timer */
1393         cancel_delayed_work_sync(&els_req->timeout_work);
1394
1395         /* Call callback function to complete command */
1396         if (els_req->cb_func && els_req->cb_arg) {
1397                 els_req->cb_func(els_req->cb_arg);
1398                 els_req->cb_arg = NULL;
1399         }
1400
1401         /* Release kref for original initiate_els */
1402         kref_put(&els_req->refcount, qedf_release_cmd);
1403 }
1404
1405 /* A value of -1 for lun is a wild card that means flush all
1406  * active SCSI I/Os for the target.
1407  */
1408 void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
1409 {
1410         struct qedf_ioreq *io_req;
1411         struct qedf_ctx *qedf;
1412         struct qedf_cmd_mgr *cmd_mgr;
1413         int i, rc;
1414
1415         if (!fcport)
1416                 return;
1417
1418         qedf = fcport->qedf;
1419         cmd_mgr = qedf->cmd_mgr;
1420
1421         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Flush active i/o's.\n");
1422
1423         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1424                 io_req = &cmd_mgr->cmds[i];
1425
1426                 if (!io_req)
1427                         continue;
1428                 if (io_req->fcport != fcport)
1429                         continue;
1430                 if (io_req->cmd_type == QEDF_ELS) {
1431                         rc = kref_get_unless_zero(&io_req->refcount);
1432                         if (!rc) {
1433                                 QEDF_ERR(&(qedf->dbg_ctx),
1434                                     "Could not get kref for io_req=0x%p.\n",
1435                                     io_req);
1436                                 continue;
1437                         }
1438                         qedf_flush_els_req(qedf, io_req);
1439                         /*
1440                          * Release the kref and go back to the top of the
1441                          * loop.
1442                          */
1443                         goto free_cmd;
1444                 }
1445
1446                 if (!io_req->sc_cmd)
1447                         continue;
1448                 if (lun > 0) {
1449                         if (io_req->sc_cmd->device->lun !=
1450                             (u64)lun)
1451                                 continue;
1452                 }
1453
1454                 /*
1455                  * Use kref_get_unless_zero in the unlikely case the command
1456                  * we're about to flush was completed in the normal SCSI path
1457                  */
1458                 rc = kref_get_unless_zero(&io_req->refcount);
1459                 if (!rc) {
1460                         QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
1461                             "io_req=0x%p\n", io_req);
1462                         continue;
1463                 }
1464                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1465                     "Cleanup xid=0x%x.\n", io_req->xid);
1466
1467                 /* Cleanup task and return I/O mid-layer */
1468                 qedf_initiate_cleanup(io_req, true);
1469
1470 free_cmd:
1471                 kref_put(&io_req->refcount, qedf_release_cmd);
1472         }
1473 }
1474
1475 /*
1476  * Initiate a ABTS middle path command. Note that we don't have to initialize
1477  * the task context for an ABTS task.
1478  */
1479 int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
1480 {
1481         struct fc_lport *lport;
1482         struct qedf_rport *fcport = io_req->fcport;
1483         struct fc_rport_priv *rdata;
1484         struct qedf_ctx *qedf;
1485         u16 xid;
1486         u32 r_a_tov = 0;
1487         int rc = 0;
1488         unsigned long flags;
1489         struct fcoe_wqe *sqe;
1490         u16 sqe_idx;
1491
1492         /* Sanity check qedf_rport before dereferencing any pointers */
1493         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1494                 QEDF_ERR(NULL, "tgt not offloaded\n");
1495                 rc = 1;
1496                 goto abts_err;
1497         }
1498
1499         rdata = fcport->rdata;
1500         r_a_tov = rdata->r_a_tov;
1501         qedf = fcport->qedf;
1502         lport = qedf->lport;
1503
1504         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
1505                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
1506                 rc = 1;
1507                 goto abts_err;
1508         }
1509
1510         if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
1511                 QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
1512                 rc = 1;
1513                 goto abts_err;
1514         }
1515
1516         /* Ensure room on SQ */
1517         if (!atomic_read(&fcport->free_sqes)) {
1518                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
1519                 rc = 1;
1520                 goto abts_err;
1521         }
1522
1523
1524         kref_get(&io_req->refcount);
1525
1526         xid = io_req->xid;
1527         qedf->control_requests++;
1528         qedf->packet_aborts++;
1529
1530         /* Set the return CPU to be the same as the request one */
1531         io_req->cpu = smp_processor_id();
1532
1533         /* Set the command type to abort */
1534         io_req->cmd_type = QEDF_ABTS;
1535         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
1536
1537         set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1538         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "ABTS io_req xid = "
1539                    "0x%x\n", xid);
1540
1541         qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT * HZ);
1542
1543         spin_lock_irqsave(&fcport->rport_lock, flags);
1544
1545         sqe_idx = qedf_get_sqe_idx(fcport);
1546         sqe = &fcport->sq[sqe_idx];
1547         memset(sqe, 0, sizeof(struct fcoe_wqe));
1548         io_req->task_params->sqe = sqe;
1549
1550         init_initiator_abort_fcoe_task(io_req->task_params);
1551         qedf_ring_doorbell(fcport);
1552
1553         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1554
1555         return rc;
1556 abts_err:
1557         /*
1558          * If the ABTS task fails to queue then we need to cleanup the
1559          * task at the firmware.
1560          */
1561         qedf_initiate_cleanup(io_req, return_scsi_cmd_on_abts);
1562         return rc;
1563 }
1564
1565 void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1566         struct qedf_ioreq *io_req)
1567 {
1568         uint32_t r_ctl;
1569         uint16_t xid;
1570
1571         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
1572                    "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
1573
1574         cancel_delayed_work(&io_req->timeout_work);
1575
1576         xid = io_req->xid;
1577         r_ctl = cqe->cqe_info.abts_info.r_ctl;
1578
1579         switch (r_ctl) {
1580         case FC_RCTL_BA_ACC:
1581                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
1582                     "ABTS response - ACC Send RRQ after R_A_TOV\n");
1583                 io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
1584                 /*
1585                  * Dont release this cmd yet. It will be relesed
1586                  * after we get RRQ response
1587                  */
1588                 kref_get(&io_req->refcount);
1589                 queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
1590                     msecs_to_jiffies(qedf->lport->r_a_tov));
1591                 break;
1592         /* For error cases let the cleanup return the command */
1593         case FC_RCTL_BA_RJT:
1594                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
1595                    "ABTS response - RJT\n");
1596                 io_req->event = QEDF_IOREQ_EV_ABORT_FAILED;
1597                 break;
1598         default:
1599                 QEDF_ERR(&(qedf->dbg_ctx), "Unknown ABTS response\n");
1600                 break;
1601         }
1602
1603         clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1604
1605         if (io_req->sc_cmd) {
1606                 if (io_req->return_scsi_cmd_on_abts)
1607                         qedf_scsi_done(qedf, io_req, DID_ERROR);
1608         }
1609
1610         /* Notify eh_abort handler that ABTS is complete */
1611         complete(&io_req->abts_done);
1612
1613         kref_put(&io_req->refcount, qedf_release_cmd);
1614 }
1615
1616 int qedf_init_mp_req(struct qedf_ioreq *io_req)
1617 {
1618         struct qedf_mp_req *mp_req;
1619         struct scsi_sge *mp_req_bd;
1620         struct scsi_sge *mp_resp_bd;
1621         struct qedf_ctx *qedf = io_req->fcport->qedf;
1622         dma_addr_t addr;
1623         uint64_t sz;
1624
1625         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_MP_REQ, "Entered.\n");
1626
1627         mp_req = (struct qedf_mp_req *)&(io_req->mp_req);
1628         memset(mp_req, 0, sizeof(struct qedf_mp_req));
1629
1630         if (io_req->cmd_type != QEDF_ELS) {
1631                 mp_req->req_len = sizeof(struct fcp_cmnd);
1632                 io_req->data_xfer_len = mp_req->req_len;
1633         } else
1634                 mp_req->req_len = io_req->data_xfer_len;
1635
1636         mp_req->req_buf = dma_alloc_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
1637             &mp_req->req_buf_dma, GFP_KERNEL);
1638         if (!mp_req->req_buf) {
1639                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req buffer\n");
1640                 qedf_free_mp_resc(io_req);
1641                 return -ENOMEM;
1642         }
1643
1644         mp_req->resp_buf = dma_alloc_coherent(&qedf->pdev->dev,
1645             QEDF_PAGE_SIZE, &mp_req->resp_buf_dma, GFP_KERNEL);
1646         if (!mp_req->resp_buf) {
1647                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc TM resp "
1648                           "buffer\n");
1649                 qedf_free_mp_resc(io_req);
1650                 return -ENOMEM;
1651         }
1652
1653         /* Allocate and map mp_req_bd and mp_resp_bd */
1654         sz = sizeof(struct scsi_sge);
1655         mp_req->mp_req_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
1656             &mp_req->mp_req_bd_dma, GFP_KERNEL);
1657         if (!mp_req->mp_req_bd) {
1658                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req bd\n");
1659                 qedf_free_mp_resc(io_req);
1660                 return -ENOMEM;
1661         }
1662
1663         mp_req->mp_resp_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
1664             &mp_req->mp_resp_bd_dma, GFP_KERNEL);
1665         if (!mp_req->mp_resp_bd) {
1666                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP resp bd\n");
1667                 qedf_free_mp_resc(io_req);
1668                 return -ENOMEM;
1669         }
1670
1671         /* Fill bd table */
1672         addr = mp_req->req_buf_dma;
1673         mp_req_bd = mp_req->mp_req_bd;
1674         mp_req_bd->sge_addr.lo = U64_LO(addr);
1675         mp_req_bd->sge_addr.hi = U64_HI(addr);
1676         mp_req_bd->sge_len = QEDF_PAGE_SIZE;
1677
1678         /*
1679          * MP buffer is either a task mgmt command or an ELS.
1680          * So the assumption is that it consumes a single bd
1681          * entry in the bd table
1682          */
1683         mp_resp_bd = mp_req->mp_resp_bd;
1684         addr = mp_req->resp_buf_dma;
1685         mp_resp_bd->sge_addr.lo = U64_LO(addr);
1686         mp_resp_bd->sge_addr.hi = U64_HI(addr);
1687         mp_resp_bd->sge_len = QEDF_PAGE_SIZE;
1688
1689         return 0;
1690 }
1691
1692 /*
1693  * Last ditch effort to clear the port if it's stuck. Used only after a
1694  * cleanup task times out.
1695  */
1696 static void qedf_drain_request(struct qedf_ctx *qedf)
1697 {
1698         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
1699                 QEDF_ERR(&(qedf->dbg_ctx), "MCP drain already active.\n");
1700                 return;
1701         }
1702
1703         /* Set bit to return all queuecommand requests as busy */
1704         set_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
1705
1706         /* Call qed drain request for function. Should be synchronous */
1707         qed_ops->common->drain(qedf->cdev);
1708
1709         /* Settle time for CQEs to be returned */
1710         msleep(100);
1711
1712         /* Unplug and continue */
1713         clear_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
1714 }
1715
1716 /*
1717  * Returns SUCCESS if the cleanup task does not timeout, otherwise return
1718  * FAILURE.
1719  */
1720 int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
1721         bool return_scsi_cmd_on_abts)
1722 {
1723         struct qedf_rport *fcport;
1724         struct qedf_ctx *qedf;
1725         uint16_t xid;
1726         struct fcoe_task_context *task;
1727         int tmo = 0;
1728         int rc = SUCCESS;
1729         unsigned long flags;
1730         struct fcoe_wqe *sqe;
1731         u16 sqe_idx;
1732
1733         fcport = io_req->fcport;
1734         if (!fcport) {
1735                 QEDF_ERR(NULL, "fcport is NULL.\n");
1736                 return SUCCESS;
1737         }
1738
1739         /* Sanity check qedf_rport before dereferencing any pointers */
1740         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1741                 QEDF_ERR(NULL, "tgt not offloaded\n");
1742                 rc = 1;
1743                 return SUCCESS;
1744         }
1745
1746         qedf = fcport->qedf;
1747         if (!qedf) {
1748                 QEDF_ERR(NULL, "qedf is NULL.\n");
1749                 return SUCCESS;
1750         }
1751
1752         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1753             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
1754                 QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
1755                           "cleanup processing or already completed.\n",
1756                           io_req->xid);
1757                 return SUCCESS;
1758         }
1759
1760         /* Ensure room on SQ */
1761         if (!atomic_read(&fcport->free_sqes)) {
1762                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
1763                 return FAILED;
1764         }
1765
1766
1767         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid=0x%x\n",
1768             io_req->xid);
1769
1770         /* Cleanup cmds re-use the same TID as the original I/O */
1771         xid = io_req->xid;
1772         io_req->cmd_type = QEDF_CLEANUP;
1773         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
1774
1775         /* Set the return CPU to be the same as the request one */
1776         io_req->cpu = smp_processor_id();
1777
1778         set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
1779
1780         task = qedf_get_task_mem(&qedf->tasks, xid);
1781
1782         init_completion(&io_req->tm_done);
1783
1784         spin_lock_irqsave(&fcport->rport_lock, flags);
1785
1786         sqe_idx = qedf_get_sqe_idx(fcport);
1787         sqe = &fcport->sq[sqe_idx];
1788         memset(sqe, 0, sizeof(struct fcoe_wqe));
1789         io_req->task_params->sqe = sqe;
1790
1791         init_initiator_cleanup_fcoe_task(io_req->task_params);
1792         qedf_ring_doorbell(fcport);
1793
1794         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1795
1796         tmo = wait_for_completion_timeout(&io_req->tm_done,
1797             QEDF_CLEANUP_TIMEOUT * HZ);
1798
1799         if (!tmo) {
1800                 rc = FAILED;
1801                 /* Timeout case */
1802                 QEDF_ERR(&(qedf->dbg_ctx), "Cleanup command timeout, "
1803                           "xid=%x.\n", io_req->xid);
1804                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
1805                 /* Issue a drain request if cleanup task times out */
1806                 QEDF_ERR(&(qedf->dbg_ctx), "Issuing MCP drain request.\n");
1807                 qedf_drain_request(qedf);
1808         }
1809
1810         if (io_req->sc_cmd) {
1811                 if (io_req->return_scsi_cmd_on_abts)
1812                         qedf_scsi_done(qedf, io_req, DID_ERROR);
1813         }
1814
1815         if (rc == SUCCESS)
1816                 io_req->event = QEDF_IOREQ_EV_CLEANUP_SUCCESS;
1817         else
1818                 io_req->event = QEDF_IOREQ_EV_CLEANUP_FAILED;
1819
1820         return rc;
1821 }
1822
1823 void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1824         struct qedf_ioreq *io_req)
1825 {
1826         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid = 0x%x\n",
1827                    io_req->xid);
1828
1829         clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
1830
1831         /* Complete so we can finish cleaning up the I/O */
1832         complete(&io_req->tm_done);
1833 }
1834
1835 static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
1836         uint8_t tm_flags)
1837 {
1838         struct qedf_ioreq *io_req;
1839         struct fcoe_task_context *task;
1840         struct qedf_ctx *qedf = fcport->qedf;
1841         struct fc_lport *lport = qedf->lport;
1842         int rc = 0;
1843         uint16_t xid;
1844         int tmo = 0;
1845         unsigned long flags;
1846         struct fcoe_wqe *sqe;
1847         u16 sqe_idx;
1848
1849         if (!sc_cmd) {
1850                 QEDF_ERR(&(qedf->dbg_ctx), "invalid arg\n");
1851                 return FAILED;
1852         }
1853
1854         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1855                 QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
1856                 rc = FAILED;
1857                 return FAILED;
1858         }
1859
1860         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "portid = 0x%x "
1861                    "tm_flags = %d\n", fcport->rdata->ids.port_id, tm_flags);
1862
1863         io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
1864         if (!io_req) {
1865                 QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
1866                 rc = -EAGAIN;
1867                 goto reset_tmf_err;
1868         }
1869
1870         /* Initialize rest of io_req fields */
1871         io_req->sc_cmd = sc_cmd;
1872         io_req->fcport = fcport;
1873         io_req->cmd_type = QEDF_TASK_MGMT_CMD;
1874
1875         /* Set the return CPU to be the same as the request one */
1876         io_req->cpu = smp_processor_id();
1877
1878         /* Set TM flags */
1879         io_req->io_req_flags = QEDF_READ;
1880         io_req->data_xfer_len = 0;
1881         io_req->tm_flags = tm_flags;
1882
1883         /* Default is to return a SCSI command when an error occurs */
1884         io_req->return_scsi_cmd_on_abts = true;
1885
1886         /* Obtain exchange id */
1887         xid = io_req->xid;
1888
1889         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "TMF io_req xid = "
1890                    "0x%x\n", xid);
1891
1892         /* Initialize task context for this IO request */
1893         task = qedf_get_task_mem(&qedf->tasks, xid);
1894
1895         init_completion(&io_req->tm_done);
1896
1897         spin_lock_irqsave(&fcport->rport_lock, flags);
1898
1899         sqe_idx = qedf_get_sqe_idx(fcport);
1900         sqe = &fcport->sq[sqe_idx];
1901         memset(sqe, 0, sizeof(struct fcoe_wqe));
1902
1903         qedf_init_task(fcport, lport, io_req, task, sqe);
1904         qedf_ring_doorbell(fcport);
1905
1906         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1907
1908         tmo = wait_for_completion_timeout(&io_req->tm_done,
1909             QEDF_TM_TIMEOUT * HZ);
1910
1911         if (!tmo) {
1912                 rc = FAILED;
1913                 QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
1914         } else {
1915                 /* Check TMF response code */
1916                 if (io_req->fcp_rsp_code == 0)
1917                         rc = SUCCESS;
1918                 else
1919                         rc = FAILED;
1920         }
1921
1922         if (tm_flags == FCP_TMF_LUN_RESET)
1923                 qedf_flush_active_ios(fcport, (int)sc_cmd->device->lun);
1924         else
1925                 qedf_flush_active_ios(fcport, -1);
1926
1927         kref_put(&io_req->refcount, qedf_release_cmd);
1928
1929         if (rc != SUCCESS) {
1930                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
1931                 rc = FAILED;
1932         } else {
1933                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
1934                 rc = SUCCESS;
1935         }
1936 reset_tmf_err:
1937         return rc;
1938 }
1939
1940 int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
1941 {
1942         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
1943         struct fc_rport_libfc_priv *rp = rport->dd_data;
1944         struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
1945         struct qedf_ctx *qedf;
1946         struct fc_lport *lport;
1947         int rc = SUCCESS;
1948         int rval;
1949
1950         rval = fc_remote_port_chkready(rport);
1951
1952         if (rval) {
1953                 QEDF_ERR(NULL, "device_reset rport not ready\n");
1954                 rc = FAILED;
1955                 goto tmf_err;
1956         }
1957
1958         if (fcport == NULL) {
1959                 QEDF_ERR(NULL, "device_reset: rport is NULL\n");
1960                 rc = FAILED;
1961                 goto tmf_err;
1962         }
1963
1964         qedf = fcport->qedf;
1965         lport = qedf->lport;
1966
1967         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
1968             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
1969                 rc = SUCCESS;
1970                 goto tmf_err;
1971         }
1972
1973         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
1974                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
1975                 rc = FAILED;
1976                 goto tmf_err;
1977         }
1978
1979         rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
1980
1981 tmf_err:
1982         return rc;
1983 }
1984
1985 void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1986         struct qedf_ioreq *io_req)
1987 {
1988         struct fcoe_cqe_rsp_info *fcp_rsp;
1989
1990         fcp_rsp = &cqe->cqe_info.rsp_info;
1991         qedf_parse_fcp_rsp(io_req, fcp_rsp);
1992
1993         io_req->sc_cmd = NULL;
1994         complete(&io_req->tm_done);
1995 }
1996
1997 void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
1998         struct fcoe_cqe *cqe)
1999 {
2000         unsigned long flags;
2001         uint16_t tmp;
2002         uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
2003         u32 payload_len, crc;
2004         struct fc_frame_header *fh;
2005         struct fc_frame *fp;
2006         struct qedf_io_work *io_work;
2007         u32 bdq_idx;
2008         void *bdq_addr;
2009
2010         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2011             "address.hi=%x address.lo=%x opaque_data.hi=%x "
2012             "opaque_data.lo=%x bdq_prod_idx=%u len=%u.\n",
2013             le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.address.hi),
2014             le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.address.lo),
2015             le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.hi),
2016             le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.lo),
2017             qedf->bdq_prod_idx, pktlen);
2018
2019         bdq_idx = le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.lo);
2020         if (bdq_idx >= QEDF_BDQ_SIZE) {
2021                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_idx is out of range %d.\n",
2022                     bdq_idx);
2023                 goto increment_prod;
2024         }
2025
2026         bdq_addr = qedf->bdq[bdq_idx].buf_addr;
2027         if (!bdq_addr) {
2028                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_addr is NULL, dropping "
2029                     "unsolicited packet.\n");
2030                 goto increment_prod;
2031         }
2032
2033         if (qedf_dump_frames) {
2034                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2035                     "BDQ frame is at addr=%p.\n", bdq_addr);
2036                 print_hex_dump(KERN_WARNING, "bdq ", DUMP_PREFIX_OFFSET, 16, 1,
2037                     (void *)bdq_addr, pktlen, false);
2038         }
2039
2040         /* Allocate frame */
2041         payload_len = pktlen - sizeof(struct fc_frame_header);
2042         fp = fc_frame_alloc(qedf->lport, payload_len);
2043         if (!fp) {
2044                 QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate fp.\n");
2045                 goto increment_prod;
2046         }
2047
2048         /* Copy data from BDQ buffer into fc_frame struct */
2049         fh = (struct fc_frame_header *)fc_frame_header_get(fp);
2050         memcpy(fh, (void *)bdq_addr, pktlen);
2051
2052         /* Initialize the frame so libfc sees it as a valid frame */
2053         crc = fcoe_fc_crc(fp);
2054         fc_frame_init(fp);
2055         fr_dev(fp) = qedf->lport;
2056         fr_sof(fp) = FC_SOF_I3;
2057         fr_eof(fp) = FC_EOF_T;
2058         fr_crc(fp) = cpu_to_le32(~crc);
2059
2060         /*
2061          * We need to return the frame back up to libfc in a non-atomic
2062          * context
2063          */
2064         io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
2065         if (!io_work) {
2066                 QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
2067                            "work for I/O completion.\n");
2068                 fc_frame_free(fp);
2069                 goto increment_prod;
2070         }
2071         memset(io_work, 0, sizeof(struct qedf_io_work));
2072
2073         INIT_WORK(&io_work->work, qedf_fp_io_handler);
2074
2075         /* Copy contents of CQE for deferred processing */
2076         memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
2077
2078         io_work->qedf = qedf;
2079         io_work->fp = fp;
2080
2081         queue_work_on(smp_processor_id(), qedf_io_wq, &io_work->work);
2082 increment_prod:
2083         spin_lock_irqsave(&qedf->hba_lock, flags);
2084
2085         /* Increment producer to let f/w know we've handled the frame */
2086         qedf->bdq_prod_idx++;
2087
2088         /* Producer index wraps at uint16_t boundary */
2089         if (qedf->bdq_prod_idx == 0xffff)
2090                 qedf->bdq_prod_idx = 0;
2091
2092         writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
2093         tmp = readw(qedf->bdq_primary_prod);
2094         writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
2095         tmp = readw(qedf->bdq_secondary_prod);
2096
2097         spin_unlock_irqrestore(&qedf->hba_lock, flags);
2098 }