1 // SPDX-License-Identifier: GPL-2.0-only
3 * QLogic Fibre Channel HBA Driver
4 * Copyright (c) 2003-2017 QLogic Corporation
7 #include <linux/scatterlist.h>
8 #include <linux/delay.h>
9 #include <linux/nvme.h>
10 #include <linux/nvme-fc.h>
11 #include <linux/blk-mq-pci.h>
12 #include <linux/blk-mq.h>
14 static struct nvme_fc_port_template qla_nvme_fc_transport;
15 static int qla_nvme_ls_reject_iocb(struct scsi_qla_host *vha,
17 struct qla_nvme_lsrjt_pt_arg *a,
18 bool is_xchg_terminate);
20 struct qla_nvme_unsol_ctx {
21 struct list_head elem;
22 struct scsi_qla_host *vha;
23 struct fc_port *fcport;
25 struct nvmefc_ls_rsp lsrsp;
26 struct nvmefc_ls_rsp *fd_rsp;
27 struct work_struct lsrsp_work;
28 struct work_struct abort_work;
29 __le32 exchange_address;
36 int qla_nvme_register_remote(struct scsi_qla_host *vha, struct fc_port *fcport)
38 struct qla_nvme_rport *rport;
39 struct nvme_fc_port_info req;
42 if (!IS_ENABLED(CONFIG_NVME_FC))
45 if (!vha->flags.nvme_enabled) {
46 ql_log(ql_log_info, vha, 0x2100,
47 "%s: Not registering target since Host NVME is not enabled\n",
52 if (!vha->nvme_local_port && qla_nvme_register_hba(vha))
55 if (!(fcport->nvme_prli_service_param &
56 (NVME_PRLI_SP_TARGET | NVME_PRLI_SP_DISCOVERY)) ||
57 (fcport->nvme_flag & NVME_FLAG_REGISTERED))
60 fcport->nvme_flag &= ~NVME_FLAG_RESETTING;
62 memset(&req, 0, sizeof(struct nvme_fc_port_info));
63 req.port_name = wwn_to_u64(fcport->port_name);
64 req.node_name = wwn_to_u64(fcport->node_name);
66 req.dev_loss_tmo = fcport->dev_loss_tmo;
68 if (fcport->nvme_prli_service_param & NVME_PRLI_SP_INITIATOR)
69 req.port_role = FC_PORT_ROLE_NVME_INITIATOR;
71 if (fcport->nvme_prli_service_param & NVME_PRLI_SP_TARGET)
72 req.port_role |= FC_PORT_ROLE_NVME_TARGET;
74 if (fcport->nvme_prli_service_param & NVME_PRLI_SP_DISCOVERY)
75 req.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
77 req.port_id = fcport->d_id.b24;
79 ql_log(ql_log_info, vha, 0x2102,
80 "%s: traddr=nn-0x%016llx:pn-0x%016llx PortID:%06x\n",
81 __func__, req.node_name, req.port_name,
84 ret = nvme_fc_register_remoteport(vha->nvme_local_port, &req,
85 &fcport->nvme_remote_port);
87 ql_log(ql_log_warn, vha, 0x212e,
88 "Failed to register remote port. Transport returned %d\n",
93 nvme_fc_set_remoteport_devloss(fcport->nvme_remote_port,
94 fcport->dev_loss_tmo);
96 if (fcport->nvme_prli_service_param & NVME_PRLI_SP_SLER)
97 ql_log(ql_log_info, vha, 0x212a,
98 "PortID:%06x Supports SLER\n", req.port_id);
100 if (fcport->nvme_prli_service_param & NVME_PRLI_SP_PI_CTRL)
101 ql_log(ql_log_info, vha, 0x212b,
102 "PortID:%06x Supports PI control\n", req.port_id);
104 rport = fcport->nvme_remote_port->private;
105 rport->fcport = fcport;
107 fcport->nvme_flag |= NVME_FLAG_REGISTERED;
111 /* Allocate a queue for NVMe traffic */
112 static int qla_nvme_alloc_queue(struct nvme_fc_local_port *lport,
113 unsigned int qidx, u16 qsize, void **handle)
115 struct scsi_qla_host *vha;
116 struct qla_hw_data *ha;
117 struct qla_qpair *qpair;
119 /* Map admin queue and 1st IO queue to index 0 */
123 vha = (struct scsi_qla_host *)lport->private;
126 ql_log(ql_log_info, vha, 0x2104,
127 "%s: handle %p, idx =%d, qsize %d\n",
128 __func__, handle, qidx, qsize);
130 if (qidx > qla_nvme_fc_transport.max_hw_queues) {
131 ql_log(ql_log_warn, vha, 0x212f,
132 "%s: Illegal qidx=%d. Max=%d\n",
133 __func__, qidx, qla_nvme_fc_transport.max_hw_queues);
137 /* Use base qpair if max_qpairs is 0 */
138 if (!ha->max_qpairs) {
139 qpair = ha->base_qpair;
141 if (ha->queue_pair_map[qidx]) {
142 *handle = ha->queue_pair_map[qidx];
143 ql_log(ql_log_info, vha, 0x2121,
144 "Returning existing qpair of %p for idx=%x\n",
149 qpair = qla2xxx_create_qpair(vha, 5, vha->vp_idx, true);
151 ql_log(ql_log_warn, vha, 0x2122,
152 "Failed to allocate qpair\n");
155 qla_adjust_iocb_limit(vha);
162 static void qla_nvme_release_fcp_cmd_kref(struct kref *kref)
164 struct srb *sp = container_of(kref, struct srb, cmd_kref);
165 struct nvme_private *priv = (struct nvme_private *)sp->priv;
166 struct nvmefc_fcp_req *fd;
167 struct srb_iocb *nvme;
173 nvme = &sp->u.iocb_cmd;
174 fd = nvme->u.nvme.desc;
176 spin_lock_irqsave(&priv->cmd_lock, flags);
179 if (priv->comp_status == QLA_SUCCESS) {
180 fd->rcv_rsplen = le16_to_cpu(nvme->u.nvme.rsp_pyld_len);
181 fd->status = NVME_SC_SUCCESS;
184 fd->transferred_length = 0;
185 fd->status = NVME_SC_INTERNAL;
187 spin_unlock_irqrestore(&priv->cmd_lock, flags);
191 qla2xxx_rel_qpair_sp(sp->qpair, sp);
194 static void qla_nvme_release_ls_cmd_kref(struct kref *kref)
196 struct srb *sp = container_of(kref, struct srb, cmd_kref);
197 struct nvme_private *priv = (struct nvme_private *)sp->priv;
198 struct nvmefc_ls_req *fd;
204 spin_lock_irqsave(&priv->cmd_lock, flags);
207 spin_unlock_irqrestore(&priv->cmd_lock, flags);
211 fd->done(fd, priv->comp_status);
216 static void qla_nvme_ls_complete(struct work_struct *work)
218 struct nvme_private *priv =
219 container_of(work, struct nvme_private, ls_work);
221 kref_put(&priv->sp->cmd_kref, qla_nvme_release_ls_cmd_kref);
224 static void qla_nvme_sp_ls_done(srb_t *sp, int res)
226 struct nvme_private *priv = sp->priv;
228 if (WARN_ON_ONCE(kref_read(&sp->cmd_kref) == 0))
234 priv->comp_status = res;
235 INIT_WORK(&priv->ls_work, qla_nvme_ls_complete);
236 schedule_work(&priv->ls_work);
239 static void qla_nvme_release_lsrsp_cmd_kref(struct kref *kref)
241 struct srb *sp = container_of(kref, struct srb, cmd_kref);
242 struct qla_nvme_unsol_ctx *uctx = sp->priv;
243 struct nvmefc_ls_rsp *fd_rsp;
251 spin_lock_irqsave(&uctx->cmd_lock, flags);
254 spin_unlock_irqrestore(&uctx->cmd_lock, flags);
256 fd_rsp = uctx->fd_rsp;
258 list_del(&uctx->elem);
260 fd_rsp->done(fd_rsp);
265 static void qla_nvme_lsrsp_complete(struct work_struct *work)
267 struct qla_nvme_unsol_ctx *uctx =
268 container_of(work, struct qla_nvme_unsol_ctx, lsrsp_work);
270 kref_put(&uctx->sp->cmd_kref, qla_nvme_release_lsrsp_cmd_kref);
273 static void qla_nvme_sp_lsrsp_done(srb_t *sp, int res)
275 struct qla_nvme_unsol_ctx *uctx = sp->priv;
277 if (WARN_ON_ONCE(kref_read(&sp->cmd_kref) == 0))
283 uctx->comp_status = res;
284 INIT_WORK(&uctx->lsrsp_work, qla_nvme_lsrsp_complete);
285 schedule_work(&uctx->lsrsp_work);
288 /* it assumed that QPair lock is held. */
289 static void qla_nvme_sp_done(srb_t *sp, int res)
291 struct nvme_private *priv = sp->priv;
293 priv->comp_status = res;
294 kref_put(&sp->cmd_kref, qla_nvme_release_fcp_cmd_kref);
299 static void qla_nvme_abort_work(struct work_struct *work)
301 struct nvme_private *priv =
302 container_of(work, struct nvme_private, abort_work);
303 srb_t *sp = priv->sp;
304 fc_port_t *fcport = sp->fcport;
305 struct qla_hw_data *ha = fcport->vha->hw;
306 int rval, abts_done_called = 1;
307 bool io_wait_for_abort_done;
310 ql_dbg(ql_dbg_io, fcport->vha, 0xffff,
311 "%s called for sp=%p, hndl=%x on fcport=%p desc=%p deleted=%d\n",
312 __func__, sp, sp->handle, fcport, sp->u.iocb_cmd.u.nvme.desc, fcport->deleted);
314 if (!ha->flags.fw_started || fcport->deleted == QLA_SESS_DELETED)
317 if (ha->flags.host_shutting_down) {
318 ql_log(ql_log_info, sp->fcport->vha, 0xffff,
319 "%s Calling done on sp: %p, type: 0x%x\n",
320 __func__, sp, sp->type);
326 * sp may not be valid after abort_command if return code is either
327 * SUCCESS or ERR_FROM_FW codes, so cache the value here.
329 io_wait_for_abort_done = ql2xabts_wait_nvme &&
330 QLA_ABTS_WAIT_ENABLED(sp);
333 rval = ha->isp_ops->abort_command(sp);
335 ql_dbg(ql_dbg_io, fcport->vha, 0x212b,
336 "%s: %s command for sp=%p, handle=%x on fcport=%p rval=%x\n",
337 __func__, (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
338 sp, handle, fcport, rval);
341 * If async tmf is enabled, the abort callback is called only on
342 * return codes QLA_SUCCESS and QLA_ERR_FROM_FW.
344 if (ql2xasynctmfenable &&
345 rval != QLA_SUCCESS && rval != QLA_ERR_FROM_FW)
346 abts_done_called = 0;
349 * Returned before decreasing kref so that I/O requests
350 * are waited until ABTS complete. This kref is decreased
351 * at qla24xx_abort_sp_done function.
353 if (abts_done_called && io_wait_for_abort_done)
356 /* kref_get was done before work was schedule. */
357 kref_put(&sp->cmd_kref, sp->put_fn);
360 static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport,
361 struct nvme_fc_remote_port *rport,
362 struct nvmefc_ls_rsp *fd_resp)
364 struct qla_nvme_unsol_ctx *uctx = container_of(fd_resp,
365 struct qla_nvme_unsol_ctx, lsrsp);
366 struct qla_nvme_rport *qla_rport = rport->private;
367 fc_port_t *fcport = qla_rport->fcport;
368 struct scsi_qla_host *vha = uctx->vha;
369 struct qla_hw_data *ha = vha->hw;
370 struct qla_nvme_lsrjt_pt_arg a;
371 struct srb_iocb *nvme;
373 int rval = QLA_FUNCTION_FAILED;
376 if (!fcport || fcport->deleted)
379 if (!ha->flags.fw_started)
382 /* Alloc SRB structure */
383 sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
387 sp->type = SRB_NVME_LS;
388 sp->name = "nvme_ls";
389 sp->done = qla_nvme_sp_lsrsp_done;
390 sp->put_fn = qla_nvme_release_lsrsp_cmd_kref;
391 sp->priv = (void *)uctx;
394 spin_lock_init(&uctx->cmd_lock);
395 nvme = &sp->u.iocb_cmd;
396 uctx->fd_rsp = fd_resp;
397 nvme->u.nvme.desc = fd_resp;
398 nvme->u.nvme.dir = 0;
400 nvme->u.nvme.timeout_sec = 0;
401 nvme->u.nvme.cmd_dma = fd_resp->rspdma;
402 nvme->u.nvme.cmd_len = cpu_to_le32(fd_resp->rsplen);
403 nvme->u.nvme.rsp_len = 0;
404 nvme->u.nvme.rsp_dma = 0;
405 nvme->u.nvme.exchange_address = uctx->exchange_address;
406 nvme->u.nvme.nport_handle = uctx->nport_handle;
407 nvme->u.nvme.ox_id = uctx->ox_id;
408 dma_sync_single_for_device(&ha->pdev->dev, nvme->u.nvme.cmd_dma,
409 fd_resp->rsplen, DMA_TO_DEVICE);
411 ql_dbg(ql_dbg_unsol, vha, 0x2122,
412 "Unsol lsreq portid=%06x %8phC exchange_address 0x%x ox_id 0x%x hdl 0x%x\n",
413 fcport->d_id.b24, fcport->port_name, uctx->exchange_address,
414 uctx->ox_id, uctx->nport_handle);
416 rval = qla2x00_start_sp(sp);
421 msleep(PURLS_MSLEEP_INTERVAL);
423 if (cnt < PURLS_RETRY_COUNT)
428 ql_dbg(ql_log_warn, vha, 0x2123,
429 "Failed to xmit Unsol ls response = %d\n", rval);
437 memset((void *)&a, 0, sizeof(a));
438 a.vp_idx = vha->vp_idx;
439 a.nport_handle = uctx->nport_handle;
440 a.xchg_address = uctx->exchange_address;
441 qla_nvme_ls_reject_iocb(vha, ha->base_qpair, &a, true);
446 static void qla_nvme_ls_abort(struct nvme_fc_local_port *lport,
447 struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
449 struct nvme_private *priv = fd->private;
452 spin_lock_irqsave(&priv->cmd_lock, flags);
454 spin_unlock_irqrestore(&priv->cmd_lock, flags);
458 if (!kref_get_unless_zero(&priv->sp->cmd_kref)) {
459 spin_unlock_irqrestore(&priv->cmd_lock, flags);
462 spin_unlock_irqrestore(&priv->cmd_lock, flags);
464 INIT_WORK(&priv->abort_work, qla_nvme_abort_work);
465 schedule_work(&priv->abort_work);
468 static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
469 struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
471 struct qla_nvme_rport *qla_rport = rport->private;
472 fc_port_t *fcport = qla_rport->fcport;
473 struct srb_iocb *nvme;
474 struct nvme_private *priv = fd->private;
475 struct scsi_qla_host *vha;
476 int rval = QLA_FUNCTION_FAILED;
477 struct qla_hw_data *ha;
480 if (!fcport || fcport->deleted)
486 if (!ha->flags.fw_started)
489 /* Alloc SRB structure */
490 sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
494 sp->type = SRB_NVME_LS;
495 sp->name = "nvme_ls";
496 sp->done = qla_nvme_sp_ls_done;
497 sp->put_fn = qla_nvme_release_ls_cmd_kref;
500 kref_init(&sp->cmd_kref);
501 spin_lock_init(&priv->cmd_lock);
502 nvme = &sp->u.iocb_cmd;
504 nvme->u.nvme.desc = fd;
505 nvme->u.nvme.dir = 0;
507 nvme->u.nvme.cmd_len = cpu_to_le32(fd->rqstlen);
508 nvme->u.nvme.rsp_len = cpu_to_le32(fd->rsplen);
509 nvme->u.nvme.rsp_dma = fd->rspdma;
510 nvme->u.nvme.timeout_sec = fd->timeout;
511 nvme->u.nvme.cmd_dma = fd->rqstdma;
512 dma_sync_single_for_device(&ha->pdev->dev, nvme->u.nvme.cmd_dma,
513 fd->rqstlen, DMA_TO_DEVICE);
515 rval = qla2x00_start_sp(sp);
516 if (rval != QLA_SUCCESS) {
517 ql_log(ql_log_warn, vha, 0x700e,
518 "qla2x00_start_sp failed = %d\n", rval);
528 static void qla_nvme_fcp_abort(struct nvme_fc_local_port *lport,
529 struct nvme_fc_remote_port *rport, void *hw_queue_handle,
530 struct nvmefc_fcp_req *fd)
532 struct nvme_private *priv = fd->private;
535 spin_lock_irqsave(&priv->cmd_lock, flags);
537 spin_unlock_irqrestore(&priv->cmd_lock, flags);
540 if (!kref_get_unless_zero(&priv->sp->cmd_kref)) {
541 spin_unlock_irqrestore(&priv->cmd_lock, flags);
544 spin_unlock_irqrestore(&priv->cmd_lock, flags);
546 INIT_WORK(&priv->abort_work, qla_nvme_abort_work);
547 schedule_work(&priv->abort_work);
550 static inline int qla2x00_start_nvme_mq(srb_t *sp)
555 struct cmd_nvme *cmd_pkt;
560 struct dsd64 *cur_dsd;
561 struct req_que *req = NULL;
562 struct rsp_que *rsp = NULL;
563 struct scsi_qla_host *vha = sp->fcport->vha;
564 struct qla_hw_data *ha = vha->hw;
565 struct qla_qpair *qpair = sp->qpair;
566 struct srb_iocb *nvme = &sp->u.iocb_cmd;
567 struct scatterlist *sgl, *sg;
568 struct nvmefc_fcp_req *fd = nvme->u.nvme.desc;
569 struct nvme_fc_cmd_iu *cmd = fd->cmdaddr;
570 uint32_t rval = QLA_SUCCESS;
572 /* Setup qpair pointers */
575 tot_dsds = fd->sg_cnt;
577 /* Acquire qpair specific lock */
578 spin_lock_irqsave(&qpair->qp_lock, flags);
580 handle = qla2xxx_get_next_handle(req);
585 req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
587 sp->iores.res_type = RESOURCE_IOCB | RESOURCE_EXCH;
588 sp->iores.exch_cnt = 1;
589 sp->iores.iocb_cnt = req_cnt;
590 if (qla_get_fw_resources(sp->qpair, &sp->iores)) {
595 if (req->cnt < (req_cnt + 2)) {
596 if (IS_SHADOW_REG_CAPABLE(ha)) {
599 cnt = rd_reg_dword_relaxed(req->req_q_out);
600 if (qla2x00_check_reg16_for_disconnect(vha, cnt)) {
606 if (req->ring_index < cnt)
607 req->cnt = cnt - req->ring_index;
609 req->cnt = req->length - (req->ring_index - cnt);
611 if (req->cnt < (req_cnt + 2)){
617 if (unlikely(!fd->sqid)) {
618 if (cmd->sqe.common.opcode == nvme_admin_async_event) {
619 nvme->u.nvme.aen_op = 1;
620 atomic_inc(&ha->nvme_active_aen_cnt);
624 /* Build command packet. */
625 req->current_outstanding_cmd = handle;
626 req->outstanding_cmds[handle] = sp;
630 cmd_pkt = (struct cmd_nvme *)req->ring_ptr;
631 cmd_pkt->handle = make_handle(req->id, handle);
633 /* Zero out remaining portion of packet. */
634 clr_ptr = (uint32_t *)cmd_pkt + 2;
635 memset(clr_ptr, 0, REQUEST_ENTRY_SIZE - 8);
637 cmd_pkt->entry_status = 0;
639 /* Update entry type to indicate Command NVME IOCB */
640 cmd_pkt->entry_type = COMMAND_NVME;
642 /* No data transfer how do we check buffer len == 0?? */
643 if (fd->io_dir == NVMEFC_FCP_READ) {
644 cmd_pkt->control_flags = cpu_to_le16(CF_READ_DATA);
645 qpair->counters.input_bytes += fd->payload_length;
646 qpair->counters.input_requests++;
647 } else if (fd->io_dir == NVMEFC_FCP_WRITE) {
648 cmd_pkt->control_flags = cpu_to_le16(CF_WRITE_DATA);
649 if ((vha->flags.nvme_first_burst) &&
650 (sp->fcport->nvme_prli_service_param &
651 NVME_PRLI_SP_FIRST_BURST)) {
652 if ((fd->payload_length <=
653 sp->fcport->nvme_first_burst_size) ||
654 (sp->fcport->nvme_first_burst_size == 0))
655 cmd_pkt->control_flags |=
656 cpu_to_le16(CF_NVME_FIRST_BURST_ENABLE);
658 qpair->counters.output_bytes += fd->payload_length;
659 qpair->counters.output_requests++;
660 } else if (fd->io_dir == 0) {
661 cmd_pkt->control_flags = 0;
664 if (sp->fcport->edif.enable && fd->io_dir != 0)
665 cmd_pkt->control_flags |= cpu_to_le16(CF_EN_EDIF);
667 /* Set BIT_13 of control flags for Async event */
668 if (vha->flags.nvme2_enabled &&
669 cmd->sqe.common.opcode == nvme_admin_async_event) {
670 cmd_pkt->control_flags |= cpu_to_le16(CF_ADMIN_ASYNC_EVENT);
674 cmd_pkt->nport_handle = cpu_to_le16(sp->fcport->loop_id);
675 cmd_pkt->port_id[0] = sp->fcport->d_id.b.al_pa;
676 cmd_pkt->port_id[1] = sp->fcport->d_id.b.area;
677 cmd_pkt->port_id[2] = sp->fcport->d_id.b.domain;
678 cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
681 cmd_pkt->nvme_rsp_dsd_len = cpu_to_le16(fd->rsplen);
682 put_unaligned_le64(fd->rspdma, &cmd_pkt->nvme_rsp_dseg_address);
685 cmd_pkt->nvme_cmnd_dseg_len = cpu_to_le16(fd->cmdlen);
686 cmd_pkt->nvme_cmnd_dseg_address = cpu_to_le64(fd->cmddma);
688 cmd_pkt->dseg_count = cpu_to_le16(tot_dsds);
689 cmd_pkt->byte_count = cpu_to_le32(fd->payload_length);
691 /* One DSD is available in the Command Type NVME IOCB */
693 cur_dsd = &cmd_pkt->nvme_dsd;
696 /* Load data segments */
697 for_each_sg(sgl, sg, tot_dsds, i) {
698 cont_a64_entry_t *cont_pkt;
700 /* Allocate additional continuation packets? */
701 if (avail_dsds == 0) {
703 * Five DSDs are available in the Continuation
707 /* Adjust ring index */
709 if (req->ring_index == req->length) {
711 req->ring_ptr = req->ring;
715 cont_pkt = (cont_a64_entry_t *)req->ring_ptr;
716 put_unaligned_le32(CONTINUE_A64_TYPE,
717 &cont_pkt->entry_type);
719 cur_dsd = cont_pkt->dsd;
720 avail_dsds = ARRAY_SIZE(cont_pkt->dsd);
723 append_dsd64(&cur_dsd, sg);
727 /* Set total entry count. */
728 cmd_pkt->entry_count = (uint8_t)req_cnt;
731 /* Adjust ring index. */
733 if (req->ring_index == req->length) {
735 req->ring_ptr = req->ring;
740 /* ignore nvme async cmd due to long timeout */
741 if (!nvme->u.nvme.aen_op)
742 sp->qpair->cmd_cnt++;
744 /* Set chip new ring index. */
745 wrt_reg_dword(req->req_q_in, req->ring_index);
747 if (vha->flags.process_response_queue &&
748 rsp->ring_ptr->signature != RESPONSE_PROCESSED)
749 qla24xx_process_response_queue(vha, rsp);
753 qla_put_fw_resources(sp->qpair, &sp->iores);
754 spin_unlock_irqrestore(&qpair->qp_lock, flags);
760 static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
761 struct nvme_fc_remote_port *rport, void *hw_queue_handle,
762 struct nvmefc_fcp_req *fd)
765 struct srb_iocb *nvme;
766 struct scsi_qla_host *vha;
767 struct qla_hw_data *ha;
770 struct qla_qpair *qpair = hw_queue_handle;
771 struct nvme_private *priv = fd->private;
772 struct qla_nvme_rport *qla_rport = rport->private;
775 /* nvme association has been torn down */
779 fcport = qla_rport->fcport;
781 if (unlikely(!qpair || !fcport || fcport->deleted))
784 if (!(fcport->nvme_flag & NVME_FLAG_REGISTERED))
790 if (test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags))
794 * If we know the dev is going away while the transport is still sending
795 * IO's return busy back to stall the IO Q. This happens when the
796 * link goes away and fw hasn't notified us yet, but IO's are being
797 * returned. If the dev comes back quickly we won't exhaust the IO
798 * retry count at the core.
800 if (fcport->nvme_flag & NVME_FLAG_RESETTING)
803 qpair = qla_mapq_nvme_select_qpair(ha, qpair);
805 /* Alloc SRB structure */
806 sp = qla2xxx_get_qpair_sp(vha, qpair, fcport, GFP_ATOMIC);
810 kref_init(&sp->cmd_kref);
811 spin_lock_init(&priv->cmd_lock);
814 sp->type = SRB_NVME_CMD;
815 sp->name = "nvme_cmd";
816 sp->done = qla_nvme_sp_done;
817 sp->put_fn = qla_nvme_release_fcp_cmd_kref;
821 nvme = &sp->u.iocb_cmd;
822 nvme->u.nvme.desc = fd;
824 rval = qla2x00_start_nvme_mq(sp);
825 if (rval != QLA_SUCCESS) {
826 ql_dbg(ql_dbg_io + ql_dbg_verbose, vha, 0x212d,
827 "qla2x00_start_nvme_mq failed = %d\n", rval);
830 qla2xxx_rel_qpair_sp(sp->qpair, sp);
836 static void qla_nvme_map_queues(struct nvme_fc_local_port *lport,
837 struct blk_mq_queue_map *map)
839 struct scsi_qla_host *vha = lport->private;
841 blk_mq_pci_map_queues(map, vha->hw->pdev, vha->irq_offset);
844 static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport)
846 struct scsi_qla_host *vha = lport->private;
848 ql_log(ql_log_info, vha, 0x210f,
849 "localport delete of %p completed.\n", vha->nvme_local_port);
850 vha->nvme_local_port = NULL;
851 complete(&vha->nvme_del_done);
854 static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
857 struct qla_nvme_rport *qla_rport = rport->private;
859 fcport = qla_rport->fcport;
860 fcport->nvme_remote_port = NULL;
861 fcport->nvme_flag &= ~NVME_FLAG_REGISTERED;
862 fcport->nvme_flag &= ~NVME_FLAG_DELETING;
863 ql_log(ql_log_info, fcport->vha, 0x2110,
864 "remoteport_delete of %p %8phN completed.\n",
865 fcport, fcport->port_name);
866 complete(&fcport->nvme_del_done);
869 static struct nvme_fc_port_template qla_nvme_fc_transport = {
870 .localport_delete = qla_nvme_localport_delete,
871 .remoteport_delete = qla_nvme_remoteport_delete,
872 .create_queue = qla_nvme_alloc_queue,
873 .delete_queue = NULL,
874 .ls_req = qla_nvme_ls_req,
875 .ls_abort = qla_nvme_ls_abort,
876 .fcp_io = qla_nvme_post_cmd,
877 .fcp_abort = qla_nvme_fcp_abort,
878 .xmt_ls_rsp = qla_nvme_xmt_ls_rsp,
879 .map_queues = qla_nvme_map_queues,
880 .max_hw_queues = DEF_NVME_HW_QUEUES,
881 .max_sgl_segments = 1024,
882 .max_dif_sgl_segments = 64,
883 .dma_boundary = 0xFFFFFFFF,
885 .remote_priv_sz = sizeof(struct qla_nvme_rport),
886 .lsrqst_priv_sz = sizeof(struct nvme_private),
887 .fcprqst_priv_sz = sizeof(struct nvme_private),
890 void qla_nvme_unregister_remote_port(struct fc_port *fcport)
894 if (!IS_ENABLED(CONFIG_NVME_FC))
897 ql_log(ql_log_warn, fcport->vha, 0x2112,
898 "%s: unregister remoteport on %p %8phN\n",
899 __func__, fcport, fcport->port_name);
901 if (test_bit(PFLG_DRIVER_REMOVING, &fcport->vha->pci_flags))
902 nvme_fc_set_remoteport_devloss(fcport->nvme_remote_port, 0);
904 init_completion(&fcport->nvme_del_done);
905 ret = nvme_fc_unregister_remoteport(fcport->nvme_remote_port);
907 ql_log(ql_log_info, fcport->vha, 0x2114,
908 "%s: Failed to unregister nvme_remote_port (%d)\n",
910 wait_for_completion(&fcport->nvme_del_done);
913 void qla_nvme_delete(struct scsi_qla_host *vha)
917 if (!IS_ENABLED(CONFIG_NVME_FC))
920 if (vha->nvme_local_port) {
921 init_completion(&vha->nvme_del_done);
922 ql_log(ql_log_info, vha, 0x2116,
923 "unregister localport=%p\n",
924 vha->nvme_local_port);
925 nv_ret = nvme_fc_unregister_localport(vha->nvme_local_port);
927 ql_log(ql_log_info, vha, 0x2115,
928 "Unregister of localport failed\n");
930 wait_for_completion(&vha->nvme_del_done);
934 int qla_nvme_register_hba(struct scsi_qla_host *vha)
936 struct nvme_fc_port_template *tmpl;
937 struct qla_hw_data *ha;
938 struct nvme_fc_port_info pinfo;
941 if (!IS_ENABLED(CONFIG_NVME_FC))
945 tmpl = &qla_nvme_fc_transport;
947 if (ql2xnvme_queues < MIN_NVME_HW_QUEUES) {
948 ql_log(ql_log_warn, vha, 0xfffd,
949 "ql2xnvme_queues=%d is lower than minimum queues: %d. Resetting ql2xnvme_queues to:%d\n",
950 ql2xnvme_queues, MIN_NVME_HW_QUEUES, DEF_NVME_HW_QUEUES);
951 ql2xnvme_queues = DEF_NVME_HW_QUEUES;
952 } else if (ql2xnvme_queues > (ha->max_qpairs - 1)) {
953 ql_log(ql_log_warn, vha, 0xfffd,
954 "ql2xnvme_queues=%d is greater than available IRQs: %d. Resetting ql2xnvme_queues to: %d\n",
955 ql2xnvme_queues, (ha->max_qpairs - 1),
956 (ha->max_qpairs - 1));
957 ql2xnvme_queues = ((ha->max_qpairs - 1));
960 qla_nvme_fc_transport.max_hw_queues =
961 min((uint8_t)(ql2xnvme_queues),
962 (uint8_t)((ha->max_qpairs - 1) ? (ha->max_qpairs - 1) : 1));
964 ql_log(ql_log_info, vha, 0xfffb,
965 "Number of NVME queues used for this port: %d\n",
966 qla_nvme_fc_transport.max_hw_queues);
968 pinfo.node_name = wwn_to_u64(vha->node_name);
969 pinfo.port_name = wwn_to_u64(vha->port_name);
970 pinfo.port_role = FC_PORT_ROLE_NVME_INITIATOR;
971 pinfo.port_id = vha->d_id.b24;
973 mutex_lock(&ha->vport_lock);
975 * Check again for nvme_local_port to see if any other thread raced
976 * with this one and finished registration.
978 if (!vha->nvme_local_port) {
979 ql_log(ql_log_info, vha, 0xffff,
980 "register_localport: host-traddr=nn-0x%llx:pn-0x%llx on portID:%x\n",
981 pinfo.node_name, pinfo.port_name, pinfo.port_id);
982 qla_nvme_fc_transport.dma_boundary = vha->host->dma_boundary;
984 ret = nvme_fc_register_localport(&pinfo, tmpl,
985 get_device(&ha->pdev->dev),
986 &vha->nvme_local_port);
987 mutex_unlock(&ha->vport_lock);
989 mutex_unlock(&ha->vport_lock);
993 ql_log(ql_log_warn, vha, 0xffff,
994 "register_localport failed: ret=%x\n", ret);
996 vha->nvme_local_port->private = vha;
1002 void qla_nvme_abort_set_option(struct abort_entry_24xx *abt, srb_t *orig_sp)
1004 struct qla_hw_data *ha;
1006 if (!(ql2xabts_wait_nvme && QLA_ABTS_WAIT_ENABLED(orig_sp)))
1009 ha = orig_sp->fcport->vha->hw;
1011 WARN_ON_ONCE(abt->options & cpu_to_le16(BIT_0));
1012 /* Use Driver Specified Retry Count */
1013 abt->options |= cpu_to_le16(AOF_ABTS_RTY_CNT);
1014 abt->drv.abts_rty_cnt = cpu_to_le16(2);
1015 /* Use specified response timeout */
1016 abt->options |= cpu_to_le16(AOF_RSP_TIMEOUT);
1017 /* set it to 2 * r_a_tov in secs */
1018 abt->drv.rsp_timeout = cpu_to_le16(2 * (ha->r_a_tov / 10));
1021 void qla_nvme_abort_process_comp_status(struct abort_entry_24xx *abt, srb_t *orig_sp)
1024 struct scsi_qla_host *vha;
1026 if (!(ql2xabts_wait_nvme && QLA_ABTS_WAIT_ENABLED(orig_sp)))
1029 vha = orig_sp->fcport->vha;
1031 comp_status = le16_to_cpu(abt->comp_status);
1032 switch (comp_status) {
1033 case CS_RESET: /* reset event aborted */
1034 case CS_ABORTED: /* IOCB was cleaned */
1035 /* N_Port handle is not currently logged in */
1037 /* N_Port handle was logged out while waiting for ABTS to complete */
1038 case CS_PORT_UNAVAILABLE:
1039 /* Firmware found that the port name changed */
1040 case CS_PORT_LOGGED_OUT:
1041 /* BA_RJT was received for the ABTS */
1042 case CS_PORT_CONFIG_CHG:
1043 ql_dbg(ql_dbg_async, vha, 0xf09d,
1044 "Abort I/O IOCB completed with error, comp_status=%x\n",
1048 /* BA_RJT was received for the ABTS */
1049 case CS_REJECT_RECEIVED:
1050 ql_dbg(ql_dbg_async, vha, 0xf09e,
1051 "BA_RJT was received for the ABTS rjt_vendorUnique = %u",
1052 abt->fw.ba_rjt_vendorUnique);
1053 ql_dbg(ql_dbg_async + ql_dbg_mbx, vha, 0xf09e,
1054 "ba_rjt_reasonCodeExpl = %u, ba_rjt_reasonCode = %u\n",
1055 abt->fw.ba_rjt_reasonCodeExpl, abt->fw.ba_rjt_reasonCode);
1059 ql_dbg(ql_dbg_async + ql_dbg_verbose, vha, 0xf09f,
1060 "IOCB request is completed successfully comp_status=%x\n",
1065 ql_dbg(ql_dbg_async, vha, 0xf0a0,
1066 "IOCB request is failed, comp_status=%x\n", comp_status);
1070 ql_dbg(ql_dbg_async, vha, 0xf0a1,
1071 "Invalid Abort IO IOCB Completion Status %x\n",
1077 inline void qla_wait_nvme_release_cmd_kref(srb_t *orig_sp)
1079 if (!(ql2xabts_wait_nvme && QLA_ABTS_WAIT_ENABLED(orig_sp)))
1081 kref_put(&orig_sp->cmd_kref, orig_sp->put_fn);
1084 static void qla_nvme_fc_format_rjt(void *buf, u8 ls_cmd, u8 reason,
1085 u8 explanation, u8 vendor)
1087 struct fcnvme_ls_rjt *rjt = buf;
1089 rjt->w0.ls_cmd = FCNVME_LSDESC_RQST;
1090 rjt->desc_list_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt));
1091 rjt->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1092 rjt->rqst.desc_len =
1093 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1094 rjt->rqst.w0.ls_cmd = ls_cmd;
1095 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1096 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1097 rjt->rjt.reason_code = reason;
1098 rjt->rjt.reason_explanation = explanation;
1099 rjt->rjt.vendor = vendor;
1102 static void qla_nvme_lsrjt_pt_iocb(struct scsi_qla_host *vha,
1103 struct pt_ls4_request *lsrjt_iocb,
1104 struct qla_nvme_lsrjt_pt_arg *a)
1106 lsrjt_iocb->entry_type = PT_LS4_REQUEST;
1107 lsrjt_iocb->entry_count = 1;
1108 lsrjt_iocb->sys_define = 0;
1109 lsrjt_iocb->entry_status = 0;
1110 lsrjt_iocb->handle = QLA_SKIP_HANDLE;
1111 lsrjt_iocb->nport_handle = a->nport_handle;
1112 lsrjt_iocb->exchange_address = a->xchg_address;
1113 lsrjt_iocb->vp_index = a->vp_idx;
1115 lsrjt_iocb->control_flags = cpu_to_le16(a->control_flags);
1117 put_unaligned_le64(a->tx_addr, &lsrjt_iocb->dsd[0].address);
1118 lsrjt_iocb->dsd[0].length = cpu_to_le32(a->tx_byte_count);
1119 lsrjt_iocb->tx_dseg_count = cpu_to_le16(1);
1120 lsrjt_iocb->tx_byte_count = cpu_to_le32(a->tx_byte_count);
1122 put_unaligned_le64(a->rx_addr, &lsrjt_iocb->dsd[1].address);
1123 lsrjt_iocb->dsd[1].length = 0;
1124 lsrjt_iocb->rx_dseg_count = 0;
1125 lsrjt_iocb->rx_byte_count = 0;
1129 qla_nvme_ls_reject_iocb(struct scsi_qla_host *vha, struct qla_qpair *qp,
1130 struct qla_nvme_lsrjt_pt_arg *a, bool is_xchg_terminate)
1132 struct pt_ls4_request *lsrjt_iocb;
1134 lsrjt_iocb = __qla2x00_alloc_iocbs(qp, NULL);
1136 ql_log(ql_log_warn, vha, 0x210e,
1137 "qla2x00_alloc_iocbs failed.\n");
1138 return QLA_FUNCTION_FAILED;
1141 if (!is_xchg_terminate) {
1142 qla_nvme_fc_format_rjt((void *)vha->hw->lsrjt.c, a->opcode,
1143 a->reason, a->explanation, 0);
1145 a->tx_byte_count = sizeof(struct fcnvme_ls_rjt);
1146 a->tx_addr = vha->hw->lsrjt.cdma;
1147 a->control_flags = CF_LS4_RESPONDER << CF_LS4_SHIFT;
1149 ql_dbg(ql_dbg_unsol, vha, 0x211f,
1150 "Sending nvme fc ls reject ox_id %04x op %04x\n",
1151 a->ox_id, a->opcode);
1152 ql_dump_buffer(ql_dbg_unsol + ql_dbg_verbose, vha, 0x210f,
1153 vha->hw->lsrjt.c, sizeof(*vha->hw->lsrjt.c));
1155 a->tx_byte_count = 0;
1156 a->control_flags = CF_LS4_RESPONDER_TERM << CF_LS4_SHIFT;
1157 ql_dbg(ql_dbg_unsol, vha, 0x2110,
1158 "Terminate nvme ls xchg 0x%x\n", a->xchg_address);
1161 qla_nvme_lsrjt_pt_iocb(vha, lsrjt_iocb, a);
1162 /* flush iocb to mem before notifying hw doorbell */
1164 qla2x00_start_iocbs(vha, qp->req);
1169 * qla2xxx_process_purls_pkt() - Pass-up Unsolicited
1170 * Received FC-NVMe Link Service pkt to nvme_fc_rcv_ls_req().
1171 * LLDD need to provide memory for response buffer, which
1172 * will be used to reference the exchange corresponding
1173 * to the LS when issuing an ls response. LLDD will have to free
1174 * response buffer in lport->ops->xmt_ls_rsp().
1176 * @vha: SCSI qla host
1177 * @item: ptr to purex_item
1180 qla2xxx_process_purls_pkt(struct scsi_qla_host *vha, struct purex_item *item)
1182 struct qla_nvme_unsol_ctx *uctx = item->purls_context;
1183 struct qla_nvme_lsrjt_pt_arg a;
1186 #if (IS_ENABLED(CONFIG_NVME_FC))
1187 ret = nvme_fc_rcv_ls_req(uctx->fcport->nvme_remote_port, &uctx->lsrsp,
1188 &item->iocb, item->size);
1191 ql_dbg(ql_dbg_unsol, vha, 0x2125, "NVMe transport ls_req failed\n");
1192 memset((void *)&a, 0, sizeof(a));
1193 a.vp_idx = vha->vp_idx;
1194 a.nport_handle = uctx->nport_handle;
1195 a.xchg_address = uctx->exchange_address;
1196 qla_nvme_ls_reject_iocb(vha, vha->hw->base_qpair, &a, true);
1197 list_del(&uctx->elem);
1202 static scsi_qla_host_t *
1203 qla2xxx_get_vha_from_vp_idx(struct qla_hw_data *ha, uint16_t vp_index)
1205 scsi_qla_host_t *base_vha, *vha, *tvp;
1206 unsigned long flags;
1208 base_vha = pci_get_drvdata(ha->pdev);
1210 if (!vp_index && !ha->num_vhosts)
1213 spin_lock_irqsave(&ha->vport_slock, flags);
1214 list_for_each_entry_safe(vha, tvp, &ha->vp_list, list) {
1215 if (vha->vp_idx == vp_index) {
1216 spin_unlock_irqrestore(&ha->vport_slock, flags);
1220 spin_unlock_irqrestore(&ha->vport_slock, flags);
1225 void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp)
1227 struct nvme_fc_remote_port *rport;
1228 struct qla_nvme_rport *qla_rport;
1229 struct qla_nvme_lsrjt_pt_arg a;
1230 struct pt_ls4_rx_unsol *p = *pkt;
1231 struct qla_nvme_unsol_ctx *uctx;
1232 struct rsp_que *rsp_q = *rsp;
1233 struct qla_hw_data *ha;
1234 scsi_qla_host_t *vha;
1235 fc_port_t *fcport = NULL;
1236 struct purex_item *item;
1237 port_id_t d_id = {0};
1240 bool xmt_reject = false;
1244 vha = qla2xxx_get_vha_from_vp_idx(ha, p->vp_index);
1246 ql_log(ql_log_warn, NULL, 0x2110, "Invalid vp index %d\n", p->vp_index);
1251 memset((void *)&a, 0, sizeof(a));
1252 opcode = (u8 *)&p->payload[0];
1253 a.opcode = opcode[3];
1254 a.vp_idx = p->vp_index;
1255 a.nport_handle = p->nport_handle;
1257 a.xchg_address = p->exchange_address;
1259 id.b.domain = p->s_id.domain;
1260 id.b.area = p->s_id.area;
1261 id.b.al_pa = p->s_id.al_pa;
1262 d_id.b.domain = p->d_id[2];
1263 d_id.b.area = p->d_id[1];
1264 d_id.b.al_pa = p->d_id[0];
1266 fcport = qla2x00_find_fcport_by_nportid(vha, &id, 0);
1268 ql_dbg(ql_dbg_unsol, vha, 0x211e,
1269 "Failed to find sid=%06x did=%06x\n",
1271 a.reason = FCNVME_RJT_RC_INV_ASSOC;
1272 a.explanation = FCNVME_RJT_EXP_NONE;
1276 rport = fcport->nvme_remote_port;
1277 qla_rport = rport->private;
1279 item = qla27xx_copy_multiple_pkt(vha, pkt, rsp, true, false);
1281 a.reason = FCNVME_RJT_RC_LOGIC;
1282 a.explanation = FCNVME_RJT_EXP_NONE;
1287 uctx = kzalloc(sizeof(*uctx), GFP_ATOMIC);
1289 ql_log(ql_log_info, vha, 0x2126, "Failed allocate memory\n");
1290 a.reason = FCNVME_RJT_RC_LOGIC;
1291 a.explanation = FCNVME_RJT_EXP_NONE;
1298 uctx->fcport = fcport;
1299 uctx->exchange_address = p->exchange_address;
1300 uctx->nport_handle = p->nport_handle;
1301 uctx->ox_id = p->ox_id;
1302 qla_rport->uctx = uctx;
1303 INIT_LIST_HEAD(&uctx->elem);
1304 list_add_tail(&uctx->elem, &fcport->unsol_ctx_head);
1305 item->purls_context = (void *)uctx;
1307 ql_dbg(ql_dbg_unsol, vha, 0x2121,
1308 "PURLS OP[%01x] size %d xchg addr 0x%x portid %06x\n",
1309 item->iocb.iocb[3], item->size, uctx->exchange_address,
1311 /* +48 0 1 2 3 4 5 6 7 8 9 A B C D E F
1312 * ----- -----------------------------------------------
1313 * 0000: 00 00 00 05 28 00 00 00 07 00 00 00 08 00 00 00
1314 * 0010: ab ec 0f cc 00 00 8d 7d 05 00 00 00 10 00 00 00
1315 * 0020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1317 ql_dump_buffer(ql_dbg_unsol + ql_dbg_verbose, vha, 0x2120,
1318 &item->iocb, item->size);
1320 qla24xx_queue_purex_item(vha, item, qla2xxx_process_purls_pkt);
1323 qla_nvme_ls_reject_iocb(vha, (*rsp)->qpair, &a, false);
1324 __qla_consume_iocb(vha, pkt, rsp);