1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016 Avago Technologies. All rights reserved.
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/parser.h>
10 #include <linux/random.h>
11 #include <uapi/scsi/fc/fc_fs.h>
12 #include <uapi/scsi/fc/fc_els.h>
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "../host/fc.h"
20 /* *************************** Data Structures/Defines ****************** */
23 #define NVMET_LS_CTX_COUNT 256
25 struct nvmet_fc_tgtport;
26 struct nvmet_fc_tgt_assoc;
28 struct nvmet_fc_ls_iod { /* for an LS RQST RCV */
29 struct nvmefc_ls_rsp *lsrsp;
30 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
32 struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */
34 struct nvmet_fc_tgtport *tgtport;
35 struct nvmet_fc_tgt_assoc *assoc;
38 union nvmefc_ls_requests *rqstbuf;
39 union nvmefc_ls_responses *rspbuf;
43 struct scatterlist sg[2];
45 struct work_struct work;
46 } __aligned(sizeof(unsigned long long));
48 struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */
49 struct nvmefc_ls_req ls_req;
51 struct nvmet_fc_tgtport *tgtport;
55 struct list_head lsreq_list; /* tgtport->ls_req_list */
60 /* desired maximum for a single sequence - if sg list allows it */
61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
63 enum nvmet_fcp_datadir {
70 struct nvmet_fc_fcp_iod {
71 struct nvmefc_tgt_fcp_req *fcpreq;
73 struct nvme_fc_cmd_iu cmdiubuf;
74 struct nvme_fc_ersp_iu rspiubuf;
76 struct scatterlist *next_sg;
77 struct scatterlist *data_sg;
80 enum nvmet_fcp_datadir io_dir;
88 struct work_struct defer_work;
90 struct nvmet_fc_tgtport *tgtport;
91 struct nvmet_fc_tgt_queue *queue;
93 struct list_head fcp_list; /* tgtport->fcp_list */
96 struct nvmet_fc_tgtport {
97 struct nvmet_fc_target_port fc_target_port;
99 struct list_head tgt_list; /* nvmet_fc_target_list */
100 struct device *dev; /* dev for dma mapping */
101 struct nvmet_fc_target_template *ops;
103 struct nvmet_fc_ls_iod *iod;
105 struct list_head ls_rcv_list;
106 struct list_head ls_req_list;
107 struct list_head ls_busylist;
108 struct list_head assoc_list;
109 struct list_head host_list;
110 struct ida assoc_cnt;
111 struct nvmet_fc_port_entry *pe;
115 struct work_struct put_work;
118 struct nvmet_fc_port_entry {
119 struct nvmet_fc_tgtport *tgtport;
120 struct nvmet_port *port;
123 struct list_head pe_list;
126 struct nvmet_fc_defer_fcp_req {
127 struct list_head req_list;
128 struct nvmefc_tgt_fcp_req *fcp_req;
131 struct nvmet_fc_tgt_queue {
142 struct nvmet_cq nvme_cq;
143 struct nvmet_sq nvme_sq;
144 struct nvmet_fc_tgt_assoc *assoc;
145 struct list_head fod_list;
146 struct list_head pending_cmd_list;
147 struct list_head avail_defer_list;
148 struct workqueue_struct *work_q;
150 /* array of fcp_iods */
151 struct nvmet_fc_fcp_iod fod[] __counted_by(sqsize);
152 } __aligned(sizeof(unsigned long long));
154 struct nvmet_fc_hostport {
155 struct nvmet_fc_tgtport *tgtport;
157 struct list_head host_list;
162 struct nvmet_fc_tgt_assoc {
165 atomic_t terminating;
166 struct nvmet_fc_tgtport *tgtport;
167 struct nvmet_fc_hostport *hostport;
168 struct nvmet_fc_ls_iod *rcv_disconn;
169 struct list_head a_list;
170 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
172 struct work_struct del_work;
177 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
179 return (iodptr - iodptr->tgtport->iod);
183 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
185 return (fodptr - fodptr->queue->fod);
190 * Association and Connection IDs:
192 * Association ID will have random number in upper 6 bytes and zero
195 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
197 * note: Association ID = Connection ID for queue 0
199 #define BYTES_FOR_QID sizeof(u16)
200 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
201 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
204 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
206 return (assoc->association_id | qid);
210 nvmet_fc_getassociationid(u64 connectionid)
212 return connectionid & ~NVMET_FC_QUEUEID_MASK;
216 nvmet_fc_getqueueid(u64 connectionid)
218 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
221 static inline struct nvmet_fc_tgtport *
222 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
224 return container_of(targetport, struct nvmet_fc_tgtport,
228 static inline struct nvmet_fc_fcp_iod *
229 nvmet_req_to_fod(struct nvmet_req *nvme_req)
231 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
235 /* *************************** Globals **************************** */
238 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
240 static LIST_HEAD(nvmet_fc_target_list);
241 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
242 static LIST_HEAD(nvmet_fc_portentry_list);
245 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
246 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
247 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
248 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
249 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
250 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
251 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
252 static void nvmet_fc_put_tgtport_work(struct work_struct *work)
254 struct nvmet_fc_tgtport *tgtport =
255 container_of(work, struct nvmet_fc_tgtport, put_work);
257 nvmet_fc_tgtport_put(tgtport);
259 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
260 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
261 struct nvmet_fc_fcp_iod *fod);
262 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
263 static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
264 struct nvmet_fc_ls_iod *iod);
267 /* *********************** FC-NVME DMA Handling **************************** */
270 * The fcloop device passes in a NULL device pointer. Real LLD's will
271 * pass in a valid device pointer. If NULL is passed to the dma mapping
272 * routines, depending on the platform, it may or may not succeed, and
276 * Wrapper all the dma routines and check the dev pointer.
278 * If simple mappings (return just a dma address, we'll noop them,
279 * returning a dma address of 0.
281 * On more complex mappings (dma_map_sg), a pseudo routine fills
282 * in the scatter list, setting all dma addresses to 0.
285 static inline dma_addr_t
286 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
287 enum dma_data_direction dir)
289 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
293 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
295 return dev ? dma_mapping_error(dev, dma_addr) : 0;
299 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
300 enum dma_data_direction dir)
303 dma_unmap_single(dev, addr, size, dir);
307 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
308 enum dma_data_direction dir)
311 dma_sync_single_for_cpu(dev, addr, size, dir);
315 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
316 enum dma_data_direction dir)
319 dma_sync_single_for_device(dev, addr, size, dir);
322 /* pseudo dma_map_sg call */
324 fc_map_sg(struct scatterlist *sg, int nents)
326 struct scatterlist *s;
329 WARN_ON(nents == 0 || sg[0].length == 0);
331 for_each_sg(sg, s, nents, i) {
333 #ifdef CONFIG_NEED_SG_DMA_LENGTH
334 s->dma_length = s->length;
341 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
342 enum dma_data_direction dir)
344 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
348 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
349 enum dma_data_direction dir)
352 dma_unmap_sg(dev, sg, nents, dir);
356 /* ********************** FC-NVME LS XMT Handling ************************* */
360 __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)
362 struct nvmet_fc_tgtport *tgtport = lsop->tgtport;
363 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
366 spin_lock_irqsave(&tgtport->lock, flags);
368 if (!lsop->req_queued) {
369 spin_unlock_irqrestore(&tgtport->lock, flags);
373 list_del(&lsop->lsreq_list);
375 lsop->req_queued = false;
377 spin_unlock_irqrestore(&tgtport->lock, flags);
379 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
380 (lsreq->rqstlen + lsreq->rsplen),
384 queue_work(nvmet_wq, &tgtport->put_work);
388 __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,
389 struct nvmet_fc_ls_req_op *lsop,
390 void (*done)(struct nvmefc_ls_req *req, int status))
392 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
396 if (!tgtport->ops->ls_req)
399 if (!nvmet_fc_tgtport_get(tgtport))
403 lsop->req_queued = false;
404 INIT_LIST_HEAD(&lsop->lsreq_list);
406 lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr,
407 lsreq->rqstlen + lsreq->rsplen,
409 if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) {
413 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
415 spin_lock_irqsave(&tgtport->lock, flags);
417 list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list);
419 lsop->req_queued = true;
421 spin_unlock_irqrestore(&tgtport->lock, flags);
423 ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle,
431 lsop->ls_error = ret;
432 spin_lock_irqsave(&tgtport->lock, flags);
433 lsop->req_queued = false;
434 list_del(&lsop->lsreq_list);
435 spin_unlock_irqrestore(&tgtport->lock, flags);
436 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
437 (lsreq->rqstlen + lsreq->rsplen),
440 nvmet_fc_tgtport_put(tgtport);
446 nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport,
447 struct nvmet_fc_ls_req_op *lsop,
448 void (*done)(struct nvmefc_ls_req *req, int status))
450 /* don't wait for completion */
452 return __nvmet_fc_send_ls_req(tgtport, lsop, done);
456 nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
458 struct nvmet_fc_ls_req_op *lsop =
459 container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req);
461 __nvmet_fc_finish_ls_req(lsop);
463 /* fc-nvme target doesn't care about success or failure of cmd */
469 * This routine sends a FC-NVME LS to disconnect (aka terminate)
470 * the FC-NVME Association. Terminating the association also
471 * terminates the FC-NVME connections (per queue, both admin and io
472 * queues) that are part of the association. E.g. things are torn
473 * down, and the related FC-NVME Association ID and Connection IDs
476 * The behavior of the fc-nvme target is such that it's
477 * understanding of the association and connections will implicitly
478 * be torn down. The action is implicit as it may be due to a loss of
479 * connectivity with the fc-nvme host, so the target may never get a
480 * response even if it tried. As such, the action of this routine
481 * is to asynchronously send the LS, ignore any results of the LS, and
482 * continue on with terminating the association. If the fc-nvme host
483 * is present and receives the LS, it too can tear down.
486 nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)
488 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
489 struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
490 struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
491 struct nvmet_fc_ls_req_op *lsop;
492 struct nvmefc_ls_req *lsreq;
496 * If ls_req is NULL or no hosthandle, it's an older lldd and no
497 * message is normal. Otherwise, send unless the hostport has
498 * already been invalidated by the lldd.
500 if (!tgtport->ops->ls_req || assoc->hostport->invalid)
503 lsop = kzalloc((sizeof(*lsop) +
504 sizeof(*discon_rqst) + sizeof(*discon_acc) +
505 tgtport->ops->lsrqst_priv_sz), GFP_KERNEL);
507 dev_info(tgtport->dev,
508 "{%d:%d} send Disconnect Association failed: ENOMEM\n",
509 tgtport->fc_target_port.port_num, assoc->a_id);
513 discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
514 discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
515 lsreq = &lsop->ls_req;
516 if (tgtport->ops->lsrqst_priv_sz)
517 lsreq->private = (void *)&discon_acc[1];
519 lsreq->private = NULL;
521 lsop->tgtport = tgtport;
522 lsop->hosthandle = assoc->hostport->hosthandle;
524 nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
525 assoc->association_id);
527 ret = nvmet_fc_send_ls_req_async(tgtport, lsop,
528 nvmet_fc_disconnect_assoc_done);
530 dev_info(tgtport->dev,
531 "{%d:%d} XMT Disconnect Association failed: %d\n",
532 tgtport->fc_target_port.port_num, assoc->a_id, ret);
538 /* *********************** FC-NVME Port Management ************************ */
542 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
544 struct nvmet_fc_ls_iod *iod;
547 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
554 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
555 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
556 iod->tgtport = tgtport;
557 list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
559 iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) +
560 sizeof(union nvmefc_ls_responses),
565 iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1];
567 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
568 sizeof(*iod->rspbuf),
570 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
578 list_del(&iod->ls_rcv_list);
579 for (iod--, i--; i >= 0; iod--, i--) {
580 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
581 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
583 list_del(&iod->ls_rcv_list);
592 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
594 struct nvmet_fc_ls_iod *iod = tgtport->iod;
597 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
598 fc_dma_unmap_single(tgtport->dev,
599 iod->rspdma, sizeof(*iod->rspbuf),
602 list_del(&iod->ls_rcv_list);
607 static struct nvmet_fc_ls_iod *
608 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
610 struct nvmet_fc_ls_iod *iod;
613 spin_lock_irqsave(&tgtport->lock, flags);
614 iod = list_first_entry_or_null(&tgtport->ls_rcv_list,
615 struct nvmet_fc_ls_iod, ls_rcv_list);
617 list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist);
618 spin_unlock_irqrestore(&tgtport->lock, flags);
624 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
625 struct nvmet_fc_ls_iod *iod)
629 spin_lock_irqsave(&tgtport->lock, flags);
630 list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
631 spin_unlock_irqrestore(&tgtport->lock, flags);
635 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
636 struct nvmet_fc_tgt_queue *queue)
638 struct nvmet_fc_fcp_iod *fod = queue->fod;
641 for (i = 0; i < queue->sqsize; fod++, i++) {
642 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
643 fod->tgtport = tgtport;
647 fod->aborted = false;
649 list_add_tail(&fod->fcp_list, &queue->fod_list);
650 spin_lock_init(&fod->flock);
652 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
653 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
654 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
655 list_del(&fod->fcp_list);
656 for (fod--, i--; i >= 0; fod--, i--) {
657 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
658 sizeof(fod->rspiubuf),
661 list_del(&fod->fcp_list);
670 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
671 struct nvmet_fc_tgt_queue *queue)
673 struct nvmet_fc_fcp_iod *fod = queue->fod;
676 for (i = 0; i < queue->sqsize; fod++, i++) {
678 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
679 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
683 static struct nvmet_fc_fcp_iod *
684 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
686 struct nvmet_fc_fcp_iod *fod;
688 lockdep_assert_held(&queue->qlock);
690 fod = list_first_entry_or_null(&queue->fod_list,
691 struct nvmet_fc_fcp_iod, fcp_list);
693 list_del(&fod->fcp_list);
696 * no queue reference is taken, as it was taken by the
697 * queue lookup just prior to the allocation. The iod
698 * will "inherit" that reference.
706 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
707 struct nvmet_fc_tgt_queue *queue,
708 struct nvmefc_tgt_fcp_req *fcpreq)
710 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
713 * put all admin cmds on hw queue id 0. All io commands go to
714 * the respective hw queue based on a modulo basis
716 fcpreq->hwqid = queue->qid ?
717 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
719 nvmet_fc_handle_fcp_rqst(tgtport, fod);
723 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
725 struct nvmet_fc_fcp_iod *fod =
726 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
728 /* Submit deferred IO for processing */
729 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
734 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
735 struct nvmet_fc_fcp_iod *fod)
737 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
738 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
739 struct nvmet_fc_defer_fcp_req *deferfcp;
742 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
743 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
745 fcpreq->nvmet_fc_private = NULL;
749 fod->aborted = false;
750 fod->writedataactive = false;
753 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
755 /* release the queue lookup reference on the completed IO */
756 nvmet_fc_tgt_q_put(queue);
758 spin_lock_irqsave(&queue->qlock, flags);
759 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
760 struct nvmet_fc_defer_fcp_req, req_list);
762 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
763 spin_unlock_irqrestore(&queue->qlock, flags);
767 /* Re-use the fod for the next pending cmd that was deferred */
768 list_del(&deferfcp->req_list);
770 fcpreq = deferfcp->fcp_req;
772 /* deferfcp can be reused for another IO at a later date */
773 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
775 spin_unlock_irqrestore(&queue->qlock, flags);
777 /* Save NVME CMD IO in fod */
778 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
780 /* Setup new fcpreq to be processed */
781 fcpreq->rspaddr = NULL;
783 fcpreq->nvmet_fc_private = fod;
784 fod->fcpreq = fcpreq;
787 /* inform LLDD IO is now being processed */
788 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
791 * Leave the queue lookup get reference taken when
792 * fod was originally allocated.
795 queue_work(queue->work_q, &fod->defer_work);
798 static struct nvmet_fc_tgt_queue *
799 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
802 struct nvmet_fc_tgt_queue *queue;
805 if (qid > NVMET_NR_QUEUES)
808 queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL);
812 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
813 assoc->tgtport->fc_target_port.port_num,
819 queue->sqsize = sqsize;
820 queue->assoc = assoc;
821 INIT_LIST_HEAD(&queue->fod_list);
822 INIT_LIST_HEAD(&queue->avail_defer_list);
823 INIT_LIST_HEAD(&queue->pending_cmd_list);
824 atomic_set(&queue->connected, 0);
825 atomic_set(&queue->sqtail, 0);
826 atomic_set(&queue->rsn, 1);
827 atomic_set(&queue->zrspcnt, 0);
828 spin_lock_init(&queue->qlock);
829 kref_init(&queue->ref);
831 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
833 ret = nvmet_sq_init(&queue->nvme_sq);
835 goto out_fail_iodlist;
837 WARN_ON(assoc->queues[qid]);
838 assoc->queues[qid] = queue;
843 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
844 destroy_workqueue(queue->work_q);
852 nvmet_fc_tgt_queue_free(struct kref *ref)
854 struct nvmet_fc_tgt_queue *queue =
855 container_of(ref, struct nvmet_fc_tgt_queue, ref);
857 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
859 destroy_workqueue(queue->work_q);
865 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
867 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
871 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
873 return kref_get_unless_zero(&queue->ref);
878 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
880 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
881 struct nvmet_fc_fcp_iod *fod = queue->fod;
882 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
887 disconnect = atomic_xchg(&queue->connected, 0);
889 /* if not connected, nothing to do */
893 spin_lock_irqsave(&queue->qlock, flags);
894 /* abort outstanding io's */
895 for (i = 0; i < queue->sqsize; fod++, i++) {
897 spin_lock(&fod->flock);
900 * only call lldd abort routine if waiting for
901 * writedata. other outstanding ops should finish
904 if (fod->writedataactive) {
906 spin_unlock(&fod->flock);
907 tgtport->ops->fcp_abort(
908 &tgtport->fc_target_port, fod->fcpreq);
910 spin_unlock(&fod->flock);
914 /* Cleanup defer'ed IOs in queue */
915 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
917 list_del(&deferfcp->req_list);
922 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
923 struct nvmet_fc_defer_fcp_req, req_list);
927 list_del(&deferfcp->req_list);
928 spin_unlock_irqrestore(&queue->qlock, flags);
930 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
933 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
936 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
939 /* release the queue lookup reference */
940 nvmet_fc_tgt_q_put(queue);
944 spin_lock_irqsave(&queue->qlock, flags);
946 spin_unlock_irqrestore(&queue->qlock, flags);
948 flush_workqueue(queue->work_q);
950 nvmet_sq_destroy(&queue->nvme_sq);
952 nvmet_fc_tgt_q_put(queue);
955 static struct nvmet_fc_tgt_queue *
956 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
959 struct nvmet_fc_tgt_assoc *assoc;
960 struct nvmet_fc_tgt_queue *queue;
961 u64 association_id = nvmet_fc_getassociationid(connection_id);
962 u16 qid = nvmet_fc_getqueueid(connection_id);
964 if (qid > NVMET_NR_QUEUES)
968 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
969 if (association_id == assoc->association_id) {
970 queue = assoc->queues[qid];
972 (!atomic_read(&queue->connected) ||
973 !nvmet_fc_tgt_q_get(queue)))
984 nvmet_fc_hostport_free(struct kref *ref)
986 struct nvmet_fc_hostport *hostport =
987 container_of(ref, struct nvmet_fc_hostport, ref);
988 struct nvmet_fc_tgtport *tgtport = hostport->tgtport;
991 spin_lock_irqsave(&tgtport->lock, flags);
992 list_del(&hostport->host_list);
993 spin_unlock_irqrestore(&tgtport->lock, flags);
994 if (tgtport->ops->host_release && hostport->invalid)
995 tgtport->ops->host_release(hostport->hosthandle);
997 nvmet_fc_tgtport_put(tgtport);
1001 nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport)
1003 kref_put(&hostport->ref, nvmet_fc_hostport_free);
1007 nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport)
1009 return kref_get_unless_zero(&hostport->ref);
1013 nvmet_fc_free_hostport(struct nvmet_fc_hostport *hostport)
1015 /* if LLDD not implemented, leave as NULL */
1016 if (!hostport || !hostport->hosthandle)
1019 nvmet_fc_hostport_put(hostport);
1022 static struct nvmet_fc_hostport *
1023 nvmet_fc_match_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1025 struct nvmet_fc_hostport *host;
1027 lockdep_assert_held(&tgtport->lock);
1029 list_for_each_entry(host, &tgtport->host_list, host_list) {
1030 if (host->hosthandle == hosthandle && !host->invalid) {
1031 if (nvmet_fc_hostport_get(host))
1039 static struct nvmet_fc_hostport *
1040 nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1042 struct nvmet_fc_hostport *newhost, *match = NULL;
1043 unsigned long flags;
1045 /* if LLDD not implemented, leave as NULL */
1050 * take reference for what will be the newly allocated hostport if
1051 * we end up using a new allocation
1053 if (!nvmet_fc_tgtport_get(tgtport))
1054 return ERR_PTR(-EINVAL);
1056 spin_lock_irqsave(&tgtport->lock, flags);
1057 match = nvmet_fc_match_hostport(tgtport, hosthandle);
1058 spin_unlock_irqrestore(&tgtport->lock, flags);
1061 /* no new allocation - release reference */
1062 nvmet_fc_tgtport_put(tgtport);
1066 newhost = kzalloc(sizeof(*newhost), GFP_KERNEL);
1068 /* no new allocation - release reference */
1069 nvmet_fc_tgtport_put(tgtport);
1070 return ERR_PTR(-ENOMEM);
1073 spin_lock_irqsave(&tgtport->lock, flags);
1074 match = nvmet_fc_match_hostport(tgtport, hosthandle);
1076 /* new allocation not needed */
1080 newhost->tgtport = tgtport;
1081 newhost->hosthandle = hosthandle;
1082 INIT_LIST_HEAD(&newhost->host_list);
1083 kref_init(&newhost->ref);
1085 list_add_tail(&newhost->host_list, &tgtport->host_list);
1087 spin_unlock_irqrestore(&tgtport->lock, flags);
1093 nvmet_fc_delete_assoc(struct nvmet_fc_tgt_assoc *assoc)
1095 nvmet_fc_delete_target_assoc(assoc);
1096 nvmet_fc_tgt_a_put(assoc);
1100 nvmet_fc_delete_assoc_work(struct work_struct *work)
1102 struct nvmet_fc_tgt_assoc *assoc =
1103 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
1104 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1106 nvmet_fc_delete_assoc(assoc);
1107 nvmet_fc_tgtport_put(tgtport);
1111 nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc *assoc)
1113 nvmet_fc_tgtport_get(assoc->tgtport);
1114 queue_work(nvmet_wq, &assoc->del_work);
1118 nvmet_fc_assoc_exits(struct nvmet_fc_tgtport *tgtport, u64 association_id)
1120 struct nvmet_fc_tgt_assoc *a;
1122 list_for_each_entry_rcu(a, &tgtport->assoc_list, a_list) {
1123 if (association_id == a->association_id)
1130 static struct nvmet_fc_tgt_assoc *
1131 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1133 struct nvmet_fc_tgt_assoc *assoc;
1134 unsigned long flags;
1142 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
1146 idx = ida_alloc(&tgtport->assoc_cnt, GFP_KERNEL);
1148 goto out_free_assoc;
1150 assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle);
1151 if (IS_ERR(assoc->hostport))
1154 assoc->tgtport = tgtport;
1156 INIT_LIST_HEAD(&assoc->a_list);
1157 kref_init(&assoc->ref);
1158 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc_work);
1159 atomic_set(&assoc->terminating, 0);
1163 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
1164 ran = ran << BYTES_FOR_QID_SHIFT;
1166 spin_lock_irqsave(&tgtport->lock, flags);
1168 if (!nvmet_fc_assoc_exits(tgtport, ran)) {
1169 assoc->association_id = ran;
1170 list_add_tail_rcu(&assoc->a_list, &tgtport->assoc_list);
1174 spin_unlock_irqrestore(&tgtport->lock, flags);
1180 ida_free(&tgtport->assoc_cnt, idx);
1187 nvmet_fc_target_assoc_free(struct kref *ref)
1189 struct nvmet_fc_tgt_assoc *assoc =
1190 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
1191 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1192 struct nvmet_fc_ls_iod *oldls;
1193 unsigned long flags;
1196 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1197 if (assoc->queues[i])
1198 nvmet_fc_delete_target_queue(assoc->queues[i]);
1201 /* Send Disconnect now that all i/o has completed */
1202 nvmet_fc_xmt_disconnect_assoc(assoc);
1204 nvmet_fc_free_hostport(assoc->hostport);
1205 spin_lock_irqsave(&tgtport->lock, flags);
1206 oldls = assoc->rcv_disconn;
1207 spin_unlock_irqrestore(&tgtport->lock, flags);
1208 /* if pending Rcv Disconnect Association LS, send rsp now */
1210 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1211 ida_free(&tgtport->assoc_cnt, assoc->a_id);
1212 dev_info(tgtport->dev,
1213 "{%d:%d} Association freed\n",
1214 tgtport->fc_target_port.port_num, assoc->a_id);
1219 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
1221 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
1225 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
1227 return kref_get_unless_zero(&assoc->ref);
1231 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
1233 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1234 unsigned long flags;
1237 terminating = atomic_xchg(&assoc->terminating, 1);
1239 /* if already terminating, do nothing */
1243 spin_lock_irqsave(&tgtport->lock, flags);
1244 list_del_rcu(&assoc->a_list);
1245 spin_unlock_irqrestore(&tgtport->lock, flags);
1249 /* ensure all in-flight I/Os have been processed */
1250 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1251 if (assoc->queues[i])
1252 flush_workqueue(assoc->queues[i]->work_q);
1255 dev_info(tgtport->dev,
1256 "{%d:%d} Association deleted\n",
1257 tgtport->fc_target_port.port_num, assoc->a_id);
1260 static struct nvmet_fc_tgt_assoc *
1261 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
1264 struct nvmet_fc_tgt_assoc *assoc;
1265 struct nvmet_fc_tgt_assoc *ret = NULL;
1268 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1269 if (association_id == assoc->association_id) {
1271 if (!nvmet_fc_tgt_a_get(assoc))
1282 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport,
1283 struct nvmet_fc_port_entry *pe,
1284 struct nvmet_port *port)
1286 lockdep_assert_held(&nvmet_fc_tgtlock);
1288 pe->tgtport = tgtport;
1294 pe->node_name = tgtport->fc_target_port.node_name;
1295 pe->port_name = tgtport->fc_target_port.port_name;
1296 INIT_LIST_HEAD(&pe->pe_list);
1298 list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list);
1302 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe)
1304 unsigned long flags;
1306 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1308 pe->tgtport->pe = NULL;
1309 list_del(&pe->pe_list);
1310 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1314 * called when a targetport deregisters. Breaks the relationship
1315 * with the nvmet port, but leaves the port_entry in place so that
1316 * re-registration can resume operation.
1319 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport)
1321 struct nvmet_fc_port_entry *pe;
1322 unsigned long flags;
1324 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1329 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1333 * called when a new targetport is registered. Looks in the
1334 * existing nvmet port_entries to see if the nvmet layer is
1335 * configured for the targetport's wwn's. (the targetport existed,
1336 * nvmet configured, the lldd unregistered the tgtport, and is now
1337 * reregistering the same targetport). If so, set the nvmet port
1338 * port entry on the targetport.
1341 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport)
1343 struct nvmet_fc_port_entry *pe;
1344 unsigned long flags;
1346 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1347 list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) {
1348 if (tgtport->fc_target_port.node_name == pe->node_name &&
1349 tgtport->fc_target_port.port_name == pe->port_name) {
1350 WARN_ON(pe->tgtport);
1352 pe->tgtport = tgtport;
1356 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1360 * nvmet_fc_register_targetport - transport entry point called by an
1361 * LLDD to register the existence of a local
1362 * NVME subystem FC port.
1363 * @pinfo: pointer to information about the port to be registered
1364 * @template: LLDD entrypoints and operational parameters for the port
1365 * @dev: physical hardware device node port corresponds to. Will be
1366 * used for DMA mappings
1367 * @portptr: pointer to a local port pointer. Upon success, the routine
1368 * will allocate a nvme_fc_local_port structure and place its
1369 * address in the local port pointer. Upon failure, local port
1370 * pointer will be set to NULL.
1373 * a completion status. Must be 0 upon success; a negative errno
1374 * (ex: -ENXIO) upon failure.
1377 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
1378 struct nvmet_fc_target_template *template,
1380 struct nvmet_fc_target_port **portptr)
1382 struct nvmet_fc_tgtport *newrec;
1383 unsigned long flags;
1386 if (!template->xmt_ls_rsp || !template->fcp_op ||
1387 !template->fcp_abort ||
1388 !template->fcp_req_release || !template->targetport_delete ||
1389 !template->max_hw_queues || !template->max_sgl_segments ||
1390 !template->max_dif_sgl_segments || !template->dma_boundary) {
1392 goto out_regtgt_failed;
1395 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
1399 goto out_regtgt_failed;
1402 idx = ida_alloc(&nvmet_fc_tgtport_cnt, GFP_KERNEL);
1405 goto out_fail_kfree;
1408 if (!get_device(dev) && dev) {
1413 newrec->fc_target_port.node_name = pinfo->node_name;
1414 newrec->fc_target_port.port_name = pinfo->port_name;
1415 if (template->target_priv_sz)
1416 newrec->fc_target_port.private = &newrec[1];
1418 newrec->fc_target_port.private = NULL;
1419 newrec->fc_target_port.port_id = pinfo->port_id;
1420 newrec->fc_target_port.port_num = idx;
1421 INIT_LIST_HEAD(&newrec->tgt_list);
1423 newrec->ops = template;
1424 spin_lock_init(&newrec->lock);
1425 INIT_LIST_HEAD(&newrec->ls_rcv_list);
1426 INIT_LIST_HEAD(&newrec->ls_req_list);
1427 INIT_LIST_HEAD(&newrec->ls_busylist);
1428 INIT_LIST_HEAD(&newrec->assoc_list);
1429 INIT_LIST_HEAD(&newrec->host_list);
1430 kref_init(&newrec->ref);
1431 ida_init(&newrec->assoc_cnt);
1432 newrec->max_sg_cnt = template->max_sgl_segments;
1433 INIT_WORK(&newrec->put_work, nvmet_fc_put_tgtport_work);
1435 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1438 goto out_free_newrec;
1441 nvmet_fc_portentry_rebind_tgt(newrec);
1443 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1444 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1445 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1447 *portptr = &newrec->fc_target_port;
1453 ida_free(&nvmet_fc_tgtport_cnt, idx);
1460 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1464 nvmet_fc_free_tgtport(struct kref *ref)
1466 struct nvmet_fc_tgtport *tgtport =
1467 container_of(ref, struct nvmet_fc_tgtport, ref);
1468 struct device *dev = tgtport->dev;
1469 unsigned long flags;
1471 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1472 list_del(&tgtport->tgt_list);
1473 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1475 nvmet_fc_free_ls_iodlist(tgtport);
1477 /* let the LLDD know we've finished tearing it down */
1478 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1480 ida_free(&nvmet_fc_tgtport_cnt,
1481 tgtport->fc_target_port.port_num);
1483 ida_destroy(&tgtport->assoc_cnt);
1491 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1493 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1497 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1499 return kref_get_unless_zero(&tgtport->ref);
1503 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1505 struct nvmet_fc_tgt_assoc *assoc;
1508 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1509 if (!nvmet_fc_tgt_a_get(assoc))
1511 nvmet_fc_schedule_delete_assoc(assoc);
1512 nvmet_fc_tgt_a_put(assoc);
1518 * nvmet_fc_invalidate_host - transport entry point called by an LLDD
1519 * to remove references to a hosthandle for LS's.
1521 * The nvmet-fc layer ensures that any references to the hosthandle
1522 * on the targetport are forgotten (set to NULL). The LLDD will
1523 * typically call this when a login with a remote host port has been
1524 * lost, thus LS's for the remote host port are no longer possible.
1526 * If an LS request is outstanding to the targetport/hosthandle (or
1527 * issued concurrently with the call to invalidate the host), the
1528 * LLDD is responsible for terminating/aborting the LS and completing
1529 * the LS request. It is recommended that these terminations/aborts
1530 * occur after calling to invalidate the host handle to avoid additional
1531 * retries by the nvmet-fc transport. The nvmet-fc transport may
1532 * continue to reference host handle while it cleans up outstanding
1533 * NVME associations. The nvmet-fc transport will call the
1534 * ops->host_release() callback to notify the LLDD that all references
1535 * are complete and the related host handle can be recovered.
1536 * Note: if there are no references, the callback may be called before
1537 * the invalidate host call returns.
1539 * @target_port: pointer to the (registered) target port that a prior
1540 * LS was received on and which supplied the transport the
1542 * @hosthandle: the handle (pointer) that represents the host port
1543 * that no longer has connectivity and that LS's should
1544 * no longer be directed to.
1547 nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,
1550 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1551 struct nvmet_fc_tgt_assoc *assoc, *next;
1552 unsigned long flags;
1553 bool noassoc = true;
1555 spin_lock_irqsave(&tgtport->lock, flags);
1556 list_for_each_entry_safe(assoc, next,
1557 &tgtport->assoc_list, a_list) {
1558 if (assoc->hostport->hosthandle != hosthandle)
1560 if (!nvmet_fc_tgt_a_get(assoc))
1562 assoc->hostport->invalid = 1;
1564 nvmet_fc_schedule_delete_assoc(assoc);
1565 nvmet_fc_tgt_a_put(assoc);
1567 spin_unlock_irqrestore(&tgtport->lock, flags);
1569 /* if there's nothing to wait for - call the callback */
1570 if (noassoc && tgtport->ops->host_release)
1571 tgtport->ops->host_release(hosthandle);
1573 EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host);
1576 * nvmet layer has called to terminate an association
1579 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1581 struct nvmet_fc_tgtport *tgtport, *next;
1582 struct nvmet_fc_tgt_assoc *assoc;
1583 struct nvmet_fc_tgt_queue *queue;
1584 unsigned long flags;
1585 bool found_ctrl = false;
1587 /* this is a bit ugly, but don't want to make locks layered */
1588 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1589 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1591 if (!nvmet_fc_tgtport_get(tgtport))
1593 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1596 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1597 queue = assoc->queues[0];
1598 if (queue && queue->nvme_sq.ctrl == ctrl) {
1599 if (nvmet_fc_tgt_a_get(assoc))
1606 nvmet_fc_tgtport_put(tgtport);
1609 nvmet_fc_schedule_delete_assoc(assoc);
1610 nvmet_fc_tgt_a_put(assoc);
1614 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1616 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1620 * nvmet_fc_unregister_targetport - transport entry point called by an
1621 * LLDD to deregister/remove a previously
1622 * registered a local NVME subsystem FC port.
1623 * @target_port: pointer to the (registered) target port that is to be
1627 * a completion status. Must be 0 upon success; a negative errno
1628 * (ex: -ENXIO) upon failure.
1631 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1633 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1635 nvmet_fc_portentry_unbind_tgt(tgtport);
1637 /* terminate any outstanding associations */
1638 __nvmet_fc_free_assocs(tgtport);
1640 flush_workqueue(nvmet_wq);
1643 * should terminate LS's as well. However, LS's will be generated
1644 * at the tail end of association termination, so they likely don't
1645 * exist yet. And even if they did, it's worthwhile to just let
1646 * them finish and targetport ref counting will clean things up.
1649 nvmet_fc_tgtport_put(tgtport);
1653 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1656 /* ********************** FC-NVME LS RCV Handling ************************* */
1660 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1661 struct nvmet_fc_ls_iod *iod)
1663 struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc;
1664 struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc;
1665 struct nvmet_fc_tgt_queue *queue;
1668 memset(acc, 0, sizeof(*acc));
1671 * FC-NVME spec changes. There are initiators sending different
1672 * lengths as padding sizes for Create Association Cmd descriptor
1674 * Accept anything of "minimum" length. Assume format per 1.15
1675 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1676 * trailing pad length is.
1678 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1679 ret = VERR_CR_ASSOC_LEN;
1680 else if (be32_to_cpu(rqst->desc_list_len) <
1681 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1682 ret = VERR_CR_ASSOC_RQST_LEN;
1683 else if (rqst->assoc_cmd.desc_tag !=
1684 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1685 ret = VERR_CR_ASSOC_CMD;
1686 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1687 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1688 ret = VERR_CR_ASSOC_CMD_LEN;
1689 else if (!rqst->assoc_cmd.ersp_ratio ||
1690 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1691 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1692 ret = VERR_ERSP_RATIO;
1695 /* new association w/ admin queue */
1696 iod->assoc = nvmet_fc_alloc_target_assoc(
1697 tgtport, iod->hosthandle);
1699 ret = VERR_ASSOC_ALLOC_FAIL;
1701 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1702 be16_to_cpu(rqst->assoc_cmd.sqsize));
1704 ret = VERR_QUEUE_ALLOC_FAIL;
1705 nvmet_fc_tgt_a_put(iod->assoc);
1711 dev_err(tgtport->dev,
1712 "Create Association LS failed: %s\n",
1713 validation_errors[ret]);
1714 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1715 sizeof(*acc), rqst->w0.ls_cmd,
1716 FCNVME_RJT_RC_LOGIC,
1717 FCNVME_RJT_EXP_NONE, 0);
1721 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1722 atomic_set(&queue->connected, 1);
1723 queue->sqhd = 0; /* best place to init value */
1725 dev_info(tgtport->dev,
1726 "{%d:%d} Association created\n",
1727 tgtport->fc_target_port.port_num, iod->assoc->a_id);
1729 /* format a response */
1731 iod->lsrsp->rsplen = sizeof(*acc);
1733 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1735 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1736 FCNVME_LS_CREATE_ASSOCIATION);
1737 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1738 acc->associd.desc_len =
1740 sizeof(struct fcnvme_lsdesc_assoc_id));
1741 acc->associd.association_id =
1742 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1743 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1744 acc->connectid.desc_len =
1746 sizeof(struct fcnvme_lsdesc_conn_id));
1747 acc->connectid.connection_id = acc->associd.association_id;
1751 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1752 struct nvmet_fc_ls_iod *iod)
1754 struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn;
1755 struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn;
1756 struct nvmet_fc_tgt_queue *queue;
1759 memset(acc, 0, sizeof(*acc));
1761 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1762 ret = VERR_CR_CONN_LEN;
1763 else if (rqst->desc_list_len !=
1765 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1766 ret = VERR_CR_CONN_RQST_LEN;
1767 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1768 ret = VERR_ASSOC_ID;
1769 else if (rqst->associd.desc_len !=
1771 sizeof(struct fcnvme_lsdesc_assoc_id)))
1772 ret = VERR_ASSOC_ID_LEN;
1773 else if (rqst->connect_cmd.desc_tag !=
1774 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1775 ret = VERR_CR_CONN_CMD;
1776 else if (rqst->connect_cmd.desc_len !=
1778 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1779 ret = VERR_CR_CONN_CMD_LEN;
1780 else if (!rqst->connect_cmd.ersp_ratio ||
1781 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1782 be16_to_cpu(rqst->connect_cmd.sqsize)))
1783 ret = VERR_ERSP_RATIO;
1787 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1788 be64_to_cpu(rqst->associd.association_id));
1790 ret = VERR_NO_ASSOC;
1792 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1793 be16_to_cpu(rqst->connect_cmd.qid),
1794 be16_to_cpu(rqst->connect_cmd.sqsize));
1796 ret = VERR_QUEUE_ALLOC_FAIL;
1798 /* release get taken in nvmet_fc_find_target_assoc */
1799 nvmet_fc_tgt_a_put(iod->assoc);
1804 dev_err(tgtport->dev,
1805 "Create Connection LS failed: %s\n",
1806 validation_errors[ret]);
1807 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1808 sizeof(*acc), rqst->w0.ls_cmd,
1809 (ret == VERR_NO_ASSOC) ?
1810 FCNVME_RJT_RC_INV_ASSOC :
1811 FCNVME_RJT_RC_LOGIC,
1812 FCNVME_RJT_EXP_NONE, 0);
1816 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1817 atomic_set(&queue->connected, 1);
1818 queue->sqhd = 0; /* best place to init value */
1820 /* format a response */
1822 iod->lsrsp->rsplen = sizeof(*acc);
1824 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1825 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1826 FCNVME_LS_CREATE_CONNECTION);
1827 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1828 acc->connectid.desc_len =
1830 sizeof(struct fcnvme_lsdesc_conn_id));
1831 acc->connectid.connection_id =
1832 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1833 be16_to_cpu(rqst->connect_cmd.qid)));
1837 * Returns true if the LS response is to be transmit
1838 * Returns false if the LS response is to be delayed
1841 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1842 struct nvmet_fc_ls_iod *iod)
1844 struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1845 &iod->rqstbuf->rq_dis_assoc;
1846 struct fcnvme_ls_disconnect_assoc_acc *acc =
1847 &iod->rspbuf->rsp_dis_assoc;
1848 struct nvmet_fc_tgt_assoc *assoc = NULL;
1849 struct nvmet_fc_ls_iod *oldls = NULL;
1850 unsigned long flags;
1853 memset(acc, 0, sizeof(*acc));
1855 ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst);
1857 /* match an active association - takes an assoc ref if !NULL */
1858 assoc = nvmet_fc_find_target_assoc(tgtport,
1859 be64_to_cpu(rqst->associd.association_id));
1862 ret = VERR_NO_ASSOC;
1865 if (ret || !assoc) {
1866 dev_err(tgtport->dev,
1867 "Disconnect LS failed: %s\n",
1868 validation_errors[ret]);
1869 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1870 sizeof(*acc), rqst->w0.ls_cmd,
1871 (ret == VERR_NO_ASSOC) ?
1872 FCNVME_RJT_RC_INV_ASSOC :
1873 FCNVME_RJT_RC_LOGIC,
1874 FCNVME_RJT_EXP_NONE, 0);
1878 /* format a response */
1880 iod->lsrsp->rsplen = sizeof(*acc);
1882 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1884 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1885 FCNVME_LS_DISCONNECT_ASSOC);
1888 * The rules for LS response says the response cannot
1889 * go back until ABTS's have been sent for all outstanding
1890 * I/O and a Disconnect Association LS has been sent.
1891 * So... save off the Disconnect LS to send the response
1892 * later. If there was a prior LS already saved, replace
1893 * it with the newer one and send a can't perform reject
1896 spin_lock_irqsave(&tgtport->lock, flags);
1897 oldls = assoc->rcv_disconn;
1898 assoc->rcv_disconn = iod;
1899 spin_unlock_irqrestore(&tgtport->lock, flags);
1902 dev_info(tgtport->dev,
1903 "{%d:%d} Multiple Disconnect Association LS's "
1905 tgtport->fc_target_port.port_num, assoc->a_id);
1906 /* overwrite good response with bogus failure */
1907 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1908 sizeof(*iod->rspbuf),
1909 /* ok to use rqst, LS is same */
1912 FCNVME_RJT_EXP_NONE, 0);
1913 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1916 nvmet_fc_schedule_delete_assoc(assoc);
1917 nvmet_fc_tgt_a_put(assoc);
1923 /* *********************** NVME Ctrl Routines **************************** */
1926 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1928 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1931 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1933 struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private;
1934 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1936 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1937 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1938 nvmet_fc_free_ls_iod(tgtport, iod);
1939 nvmet_fc_tgtport_put(tgtport);
1943 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1944 struct nvmet_fc_ls_iod *iod)
1948 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1949 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1951 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp);
1953 nvmet_fc_xmt_ls_rsp_done(iod->lsrsp);
1957 * Actual processing routine for received FC-NVME LS Requests from the LLD
1960 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1961 struct nvmet_fc_ls_iod *iod)
1963 struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0;
1964 bool sendrsp = true;
1966 iod->lsrsp->nvme_fc_private = iod;
1967 iod->lsrsp->rspbuf = iod->rspbuf;
1968 iod->lsrsp->rspdma = iod->rspdma;
1969 iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done;
1970 /* Be preventative. handlers will later set to valid length */
1971 iod->lsrsp->rsplen = 0;
1977 * parse request input, execute the request, and format the
1980 switch (w0->ls_cmd) {
1981 case FCNVME_LS_CREATE_ASSOCIATION:
1982 /* Creates Association and initial Admin Queue/Connection */
1983 nvmet_fc_ls_create_association(tgtport, iod);
1985 case FCNVME_LS_CREATE_CONNECTION:
1986 /* Creates an IO Queue/Connection */
1987 nvmet_fc_ls_create_connection(tgtport, iod);
1989 case FCNVME_LS_DISCONNECT_ASSOC:
1990 /* Terminate a Queue/Connection or the Association */
1991 sendrsp = nvmet_fc_ls_disconnect(tgtport, iod);
1994 iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf,
1995 sizeof(*iod->rspbuf), w0->ls_cmd,
1996 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
2000 nvmet_fc_xmt_ls_rsp(tgtport, iod);
2004 * Actual processing routine for received FC-NVME LS Requests from the LLD
2007 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
2009 struct nvmet_fc_ls_iod *iod =
2010 container_of(work, struct nvmet_fc_ls_iod, work);
2011 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
2013 nvmet_fc_handle_ls_rqst(tgtport, iod);
2018 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
2019 * upon the reception of a NVME LS request.
2021 * The nvmet-fc layer will copy payload to an internal structure for
2022 * processing. As such, upon completion of the routine, the LLDD may
2023 * immediately free/reuse the LS request buffer passed in the call.
2025 * If this routine returns error, the LLDD should abort the exchange.
2027 * @target_port: pointer to the (registered) target port the LS was
2029 * @hosthandle: pointer to the host specific data, gets stored in iod.
2030 * @lsrsp: pointer to a lsrsp structure to be used to reference
2031 * the exchange corresponding to the LS.
2032 * @lsreqbuf: pointer to the buffer containing the LS Request
2033 * @lsreqbuf_len: length, in bytes, of the received LS request
2036 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
2038 struct nvmefc_ls_rsp *lsrsp,
2039 void *lsreqbuf, u32 lsreqbuf_len)
2041 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2042 struct nvmet_fc_ls_iod *iod;
2043 struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
2045 if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
2046 dev_info(tgtport->dev,
2047 "RCV %s LS failed: payload too large (%d)\n",
2048 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2049 nvmefc_ls_names[w0->ls_cmd] : "",
2054 if (!nvmet_fc_tgtport_get(tgtport)) {
2055 dev_info(tgtport->dev,
2056 "RCV %s LS failed: target deleting\n",
2057 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2058 nvmefc_ls_names[w0->ls_cmd] : "");
2062 iod = nvmet_fc_alloc_ls_iod(tgtport);
2064 dev_info(tgtport->dev,
2065 "RCV %s LS failed: context allocation failed\n",
2066 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2067 nvmefc_ls_names[w0->ls_cmd] : "");
2068 nvmet_fc_tgtport_put(tgtport);
2074 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
2075 iod->rqstdatalen = lsreqbuf_len;
2076 iod->hosthandle = hosthandle;
2078 queue_work(nvmet_wq, &iod->work);
2082 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
2086 * **********************
2087 * Start of FCP handling
2088 * **********************
2092 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2094 struct scatterlist *sg;
2097 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
2102 fod->data_sg_cnt = nent;
2103 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
2104 ((fod->io_dir == NVMET_FCP_WRITE) ?
2105 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2106 /* note: write from initiator perspective */
2107 fod->next_sg = fod->data_sg;
2112 return NVME_SC_INTERNAL;
2116 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2118 if (!fod->data_sg || !fod->data_sg_cnt)
2121 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
2122 ((fod->io_dir == NVMET_FCP_WRITE) ?
2123 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2124 sgl_free(fod->data_sg);
2125 fod->data_sg = NULL;
2126 fod->data_sg_cnt = 0;
2131 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
2135 /* egad, this is ugly. And sqtail is just a best guess */
2136 sqtail = atomic_read(&q->sqtail) % q->sqsize;
2138 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
2139 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
2144 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
2147 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2148 struct nvmet_fc_fcp_iod *fod)
2150 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
2151 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2152 struct nvme_completion *cqe = &ersp->cqe;
2153 u32 *cqewd = (u32 *)cqe;
2154 bool send_ersp = false;
2155 u32 rsn, rspcnt, xfr_length;
2157 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
2158 xfr_length = fod->req.transfer_len;
2160 xfr_length = fod->offset;
2163 * check to see if we can send a 0's rsp.
2164 * Note: to send a 0's response, the NVME-FC host transport will
2165 * recreate the CQE. The host transport knows: sq id, SQHD (last
2166 * seen in an ersp), and command_id. Thus it will create a
2167 * zero-filled CQE with those known fields filled in. Transport
2168 * must send an ersp for any condition where the cqe won't match
2171 * Here are the FC-NVME mandated cases where we must send an ersp:
2172 * every N responses, where N=ersp_ratio
2173 * force fabric commands to send ersp's (not in FC-NVME but good
2175 * normal cmds: any time status is non-zero, or status is zero
2176 * but words 0 or 1 are non-zero.
2177 * the SQ is 90% or more full
2178 * the cmd is a fused command
2179 * transferred data length not equal to cmd iu length
2181 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
2182 if (!(rspcnt % fod->queue->ersp_ratio) ||
2183 nvme_is_fabrics((struct nvme_command *) sqe) ||
2184 xfr_length != fod->req.transfer_len ||
2185 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
2186 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
2187 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
2190 /* re-set the fields */
2191 fod->fcpreq->rspaddr = ersp;
2192 fod->fcpreq->rspdma = fod->rspdma;
2195 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
2196 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
2198 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
2199 rsn = atomic_inc_return(&fod->queue->rsn);
2200 ersp->rsn = cpu_to_be32(rsn);
2201 ersp->xfrd_len = cpu_to_be32(xfr_length);
2202 fod->fcpreq->rsplen = sizeof(*ersp);
2205 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
2206 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
2209 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
2212 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
2213 struct nvmet_fc_fcp_iod *fod)
2215 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2217 /* data no longer needed */
2218 nvmet_fc_free_tgt_pgs(fod);
2221 * if an ABTS was received or we issued the fcp_abort early
2222 * don't call abort routine again.
2224 /* no need to take lock - lock was taken earlier to get here */
2226 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
2228 nvmet_fc_free_fcp_iod(fod->queue, fod);
2232 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2233 struct nvmet_fc_fcp_iod *fod)
2237 fod->fcpreq->op = NVMET_FCOP_RSP;
2238 fod->fcpreq->timeout = 0;
2240 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2242 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2244 nvmet_fc_abort_op(tgtport, fod);
2248 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
2249 struct nvmet_fc_fcp_iod *fod, u8 op)
2251 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2252 struct scatterlist *sg = fod->next_sg;
2253 unsigned long flags;
2254 u32 remaininglen = fod->req.transfer_len - fod->offset;
2259 fcpreq->offset = fod->offset;
2260 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
2263 * for next sequence:
2264 * break at a sg element boundary
2265 * attempt to keep sequence length capped at
2266 * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
2267 * be longer if a single sg element is larger
2268 * than that amount. This is done to avoid creating
2269 * a new sg list to use for the tgtport api.
2273 while (tlen < remaininglen &&
2274 fcpreq->sg_cnt < tgtport->max_sg_cnt &&
2275 tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
2277 tlen += sg_dma_len(sg);
2280 if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
2282 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
2285 if (tlen < remaininglen)
2288 fod->next_sg = NULL;
2290 fcpreq->transfer_length = tlen;
2291 fcpreq->transferred_length = 0;
2292 fcpreq->fcp_error = 0;
2296 * If the last READDATA request: check if LLDD supports
2297 * combined xfr with response.
2299 if ((op == NVMET_FCOP_READDATA) &&
2300 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
2301 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
2302 fcpreq->op = NVMET_FCOP_READDATA_RSP;
2303 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2306 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2309 * should be ok to set w/o lock as its in the thread of
2310 * execution (not an async timer routine) and doesn't
2311 * contend with any clearing action
2315 if (op == NVMET_FCOP_WRITEDATA) {
2316 spin_lock_irqsave(&fod->flock, flags);
2317 fod->writedataactive = false;
2318 spin_unlock_irqrestore(&fod->flock, flags);
2319 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2320 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
2321 fcpreq->fcp_error = ret;
2322 fcpreq->transferred_length = 0;
2323 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
2329 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
2331 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2332 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2334 /* if in the middle of an io and we need to tear down */
2336 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
2337 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2341 nvmet_fc_abort_op(tgtport, fod);
2349 * actual done handler for FCP operations when completed by the lldd
2352 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
2354 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2355 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2356 unsigned long flags;
2359 spin_lock_irqsave(&fod->flock, flags);
2361 fod->writedataactive = false;
2362 spin_unlock_irqrestore(&fod->flock, flags);
2364 switch (fcpreq->op) {
2366 case NVMET_FCOP_WRITEDATA:
2367 if (__nvmet_fc_fod_op_abort(fod, abort))
2369 if (fcpreq->fcp_error ||
2370 fcpreq->transferred_length != fcpreq->transfer_length) {
2371 spin_lock_irqsave(&fod->flock, flags);
2373 spin_unlock_irqrestore(&fod->flock, flags);
2375 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2379 fod->offset += fcpreq->transferred_length;
2380 if (fod->offset != fod->req.transfer_len) {
2381 spin_lock_irqsave(&fod->flock, flags);
2382 fod->writedataactive = true;
2383 spin_unlock_irqrestore(&fod->flock, flags);
2385 /* transfer the next chunk */
2386 nvmet_fc_transfer_fcp_data(tgtport, fod,
2387 NVMET_FCOP_WRITEDATA);
2391 /* data transfer complete, resume with nvmet layer */
2392 fod->req.execute(&fod->req);
2395 case NVMET_FCOP_READDATA:
2396 case NVMET_FCOP_READDATA_RSP:
2397 if (__nvmet_fc_fod_op_abort(fod, abort))
2399 if (fcpreq->fcp_error ||
2400 fcpreq->transferred_length != fcpreq->transfer_length) {
2401 nvmet_fc_abort_op(tgtport, fod);
2407 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2408 /* data no longer needed */
2409 nvmet_fc_free_tgt_pgs(fod);
2410 nvmet_fc_free_fcp_iod(fod->queue, fod);
2414 fod->offset += fcpreq->transferred_length;
2415 if (fod->offset != fod->req.transfer_len) {
2416 /* transfer the next chunk */
2417 nvmet_fc_transfer_fcp_data(tgtport, fod,
2418 NVMET_FCOP_READDATA);
2422 /* data transfer complete, send response */
2424 /* data no longer needed */
2425 nvmet_fc_free_tgt_pgs(fod);
2427 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2431 case NVMET_FCOP_RSP:
2432 if (__nvmet_fc_fod_op_abort(fod, abort))
2434 nvmet_fc_free_fcp_iod(fod->queue, fod);
2443 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2445 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2447 nvmet_fc_fod_op_done(fod);
2451 * actual completion handler after execution by the nvmet layer
2454 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2455 struct nvmet_fc_fcp_iod *fod, int status)
2457 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2458 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2459 unsigned long flags;
2462 spin_lock_irqsave(&fod->flock, flags);
2464 spin_unlock_irqrestore(&fod->flock, flags);
2466 /* if we have a CQE, snoop the last sq_head value */
2468 fod->queue->sqhd = cqe->sq_head;
2471 nvmet_fc_abort_op(tgtport, fod);
2475 /* if an error handling the cmd post initial parsing */
2477 /* fudge up a failed CQE status for our transport error */
2478 memset(cqe, 0, sizeof(*cqe));
2479 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2480 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2481 cqe->command_id = sqe->command_id;
2482 cqe->status = cpu_to_le16(status);
2486 * try to push the data even if the SQE status is non-zero.
2487 * There may be a status where data still was intended to
2490 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2491 /* push the data over before sending rsp */
2492 nvmet_fc_transfer_fcp_data(tgtport, fod,
2493 NVMET_FCOP_READDATA);
2497 /* writes & no data - fall thru */
2500 /* data no longer needed */
2501 nvmet_fc_free_tgt_pgs(fod);
2503 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2508 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2510 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2511 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2513 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2518 * Actual processing routine for received FC-NVME I/O Requests from the LLD
2521 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2522 struct nvmet_fc_fcp_iod *fod)
2524 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2525 u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2529 * Fused commands are currently not supported in the linux
2532 * As such, the implementation of the FC transport does not
2533 * look at the fused commands and order delivery to the upper
2534 * layer until we have both based on csn.
2537 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2539 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2540 fod->io_dir = NVMET_FCP_WRITE;
2541 if (!nvme_is_write(&cmdiu->sqe))
2542 goto transport_error;
2543 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2544 fod->io_dir = NVMET_FCP_READ;
2545 if (nvme_is_write(&cmdiu->sqe))
2546 goto transport_error;
2548 fod->io_dir = NVMET_FCP_NODATA;
2550 goto transport_error;
2553 fod->req.cmd = &fod->cmdiubuf.sqe;
2554 fod->req.cqe = &fod->rspiubuf.cqe;
2556 goto transport_error;
2557 fod->req.port = tgtport->pe->port;
2559 /* clear any response payload */
2560 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2562 fod->data_sg = NULL;
2563 fod->data_sg_cnt = 0;
2565 ret = nvmet_req_init(&fod->req,
2566 &fod->queue->nvme_cq,
2567 &fod->queue->nvme_sq,
2568 &nvmet_fc_tgt_fcp_ops);
2570 /* bad SQE content or invalid ctrl state */
2571 /* nvmet layer has already called op done to send rsp. */
2575 fod->req.transfer_len = xfrlen;
2577 /* keep a running counter of tail position */
2578 atomic_inc(&fod->queue->sqtail);
2580 if (fod->req.transfer_len) {
2581 ret = nvmet_fc_alloc_tgt_pgs(fod);
2583 nvmet_req_complete(&fod->req, ret);
2587 fod->req.sg = fod->data_sg;
2588 fod->req.sg_cnt = fod->data_sg_cnt;
2591 if (fod->io_dir == NVMET_FCP_WRITE) {
2592 /* pull the data over before invoking nvmet layer */
2593 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2600 * can invoke the nvmet_layer now. If read data, cmd completion will
2603 fod->req.execute(&fod->req);
2607 nvmet_fc_abort_op(tgtport, fod);
2611 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2612 * upon the reception of a NVME FCP CMD IU.
2614 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2615 * layer for processing.
2617 * The nvmet_fc layer allocates a local job structure (struct
2618 * nvmet_fc_fcp_iod) from the queue for the io and copies the
2619 * CMD IU buffer to the job structure. As such, on a successful
2620 * completion (returns 0), the LLDD may immediately free/reuse
2621 * the CMD IU buffer passed in the call.
2623 * However, in some circumstances, due to the packetized nature of FC
2624 * and the api of the FC LLDD which may issue a hw command to send the
2625 * response, but the LLDD may not get the hw completion for that command
2626 * and upcall the nvmet_fc layer before a new command may be
2627 * asynchronously received - its possible for a command to be received
2628 * before the LLDD and nvmet_fc have recycled the job structure. It gives
2629 * the appearance of more commands received than fits in the sq.
2630 * To alleviate this scenario, a temporary queue is maintained in the
2631 * transport for pending LLDD requests waiting for a queue job structure.
2632 * In these "overrun" cases, a temporary queue element is allocated
2633 * the LLDD request and CMD iu buffer information remembered, and the
2634 * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2635 * structure is freed, it is immediately reallocated for anything on the
2636 * pending request list. The LLDDs defer_rcv() callback is called,
2637 * informing the LLDD that it may reuse the CMD IU buffer, and the io
2638 * is then started normally with the transport.
2640 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2641 * the completion as successful but must not reuse the CMD IU buffer
2642 * until the LLDD's defer_rcv() callback has been called for the
2643 * corresponding struct nvmefc_tgt_fcp_req pointer.
2645 * If there is any other condition in which an error occurs, the
2646 * transport will return a non-zero status indicating the error.
2647 * In all cases other than -EOVERFLOW, the transport has not accepted the
2648 * request and the LLDD should abort the exchange.
2650 * @target_port: pointer to the (registered) target port the FCP CMD IU
2652 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2653 * the exchange corresponding to the FCP Exchange.
2654 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2655 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2658 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2659 struct nvmefc_tgt_fcp_req *fcpreq,
2660 void *cmdiubuf, u32 cmdiubuf_len)
2662 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2663 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2664 struct nvmet_fc_tgt_queue *queue;
2665 struct nvmet_fc_fcp_iod *fod;
2666 struct nvmet_fc_defer_fcp_req *deferfcp;
2667 unsigned long flags;
2669 /* validate iu, so the connection id can be used to find the queue */
2670 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2671 (cmdiu->format_id != NVME_CMD_FORMAT_ID) ||
2672 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2673 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2676 queue = nvmet_fc_find_target_queue(tgtport,
2677 be64_to_cpu(cmdiu->connection_id));
2682 * note: reference taken by find_target_queue
2683 * After successful fod allocation, the fod will inherit the
2684 * ownership of that reference and will remove the reference
2685 * when the fod is freed.
2688 spin_lock_irqsave(&queue->qlock, flags);
2690 fod = nvmet_fc_alloc_fcp_iod(queue);
2692 spin_unlock_irqrestore(&queue->qlock, flags);
2694 fcpreq->nvmet_fc_private = fod;
2695 fod->fcpreq = fcpreq;
2697 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2699 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2704 if (!tgtport->ops->defer_rcv) {
2705 spin_unlock_irqrestore(&queue->qlock, flags);
2706 /* release the queue lookup reference */
2707 nvmet_fc_tgt_q_put(queue);
2711 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2712 struct nvmet_fc_defer_fcp_req, req_list);
2714 /* Just re-use one that was previously allocated */
2715 list_del(&deferfcp->req_list);
2717 spin_unlock_irqrestore(&queue->qlock, flags);
2719 /* Now we need to dynamically allocate one */
2720 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2722 /* release the queue lookup reference */
2723 nvmet_fc_tgt_q_put(queue);
2726 spin_lock_irqsave(&queue->qlock, flags);
2729 /* For now, use rspaddr / rsplen to save payload information */
2730 fcpreq->rspaddr = cmdiubuf;
2731 fcpreq->rsplen = cmdiubuf_len;
2732 deferfcp->fcp_req = fcpreq;
2734 /* defer processing till a fod becomes available */
2735 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2737 /* NOTE: the queue lookup reference is still valid */
2739 spin_unlock_irqrestore(&queue->qlock, flags);
2743 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2746 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2747 * upon the reception of an ABTS for a FCP command
2749 * Notify the transport that an ABTS has been received for a FCP command
2750 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2751 * LLDD believes the command is still being worked on
2752 * (template_ops->fcp_req_release() has not been called).
2754 * The transport will wait for any outstanding work (an op to the LLDD,
2755 * which the lldd should complete with error due to the ABTS; or the
2756 * completion from the nvmet layer of the nvme command), then will
2757 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2758 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2759 * to the ABTS either after return from this function (assuming any
2760 * outstanding op work has been terminated) or upon the callback being
2763 * @target_port: pointer to the (registered) target port the FCP CMD IU
2765 * @fcpreq: pointer to the fcpreq request structure that corresponds
2766 * to the exchange that received the ABTS.
2769 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2770 struct nvmefc_tgt_fcp_req *fcpreq)
2772 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2773 struct nvmet_fc_tgt_queue *queue;
2774 unsigned long flags;
2776 if (!fod || fod->fcpreq != fcpreq)
2777 /* job appears to have already completed, ignore abort */
2782 spin_lock_irqsave(&queue->qlock, flags);
2785 * mark as abort. The abort handler, invoked upon completion
2786 * of any work, will detect the aborted status and do the
2789 spin_lock(&fod->flock);
2791 fod->aborted = true;
2792 spin_unlock(&fod->flock);
2794 spin_unlock_irqrestore(&queue->qlock, flags);
2796 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2799 struct nvmet_fc_traddr {
2805 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2809 if (match_u64(sstr, &token64))
2817 * This routine validates and extracts the WWN's from the TRADDR string.
2818 * As kernel parsers need the 0x to determine number base, universally
2819 * build string to parse with 0x prefix before parsing name strings.
2822 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2824 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2825 substring_t wwn = { name, &name[sizeof(name)-1] };
2826 int nnoffset, pnoffset;
2828 /* validate if string is one of the 2 allowed formats */
2829 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2830 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2831 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2832 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2833 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2834 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2835 NVME_FC_TRADDR_OXNNLEN;
2836 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2837 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2838 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2839 "pn-", NVME_FC_TRADDR_NNLEN))) {
2840 nnoffset = NVME_FC_TRADDR_NNLEN;
2841 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2847 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2849 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2850 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2853 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2854 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2860 pr_warn("%s: bad traddr string\n", __func__);
2865 nvmet_fc_add_port(struct nvmet_port *port)
2867 struct nvmet_fc_tgtport *tgtport;
2868 struct nvmet_fc_port_entry *pe;
2869 struct nvmet_fc_traddr traddr = { 0L, 0L };
2870 unsigned long flags;
2873 /* validate the address info */
2874 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2875 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2878 /* map the traddr address info to a target port */
2880 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2881 sizeof(port->disc_addr.traddr));
2885 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2890 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2891 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2892 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2893 (tgtport->fc_target_port.port_name == traddr.pn)) {
2894 /* a FC port can only be 1 nvmet port id */
2896 nvmet_fc_portentry_bind(tgtport, pe, port);
2903 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2912 nvmet_fc_remove_port(struct nvmet_port *port)
2914 struct nvmet_fc_port_entry *pe = port->priv;
2916 nvmet_fc_portentry_unbind(pe);
2918 /* terminate any outstanding associations */
2919 __nvmet_fc_free_assocs(pe->tgtport);
2925 nvmet_fc_discovery_chg(struct nvmet_port *port)
2927 struct nvmet_fc_port_entry *pe = port->priv;
2928 struct nvmet_fc_tgtport *tgtport = pe->tgtport;
2930 if (tgtport && tgtport->ops->discovery_event)
2931 tgtport->ops->discovery_event(&tgtport->fc_target_port);
2934 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2935 .owner = THIS_MODULE,
2936 .type = NVMF_TRTYPE_FC,
2938 .add_port = nvmet_fc_add_port,
2939 .remove_port = nvmet_fc_remove_port,
2940 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2941 .delete_ctrl = nvmet_fc_delete_ctrl,
2942 .discovery_chg = nvmet_fc_discovery_chg,
2945 static int __init nvmet_fc_init_module(void)
2947 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2950 static void __exit nvmet_fc_exit_module(void)
2952 /* ensure any shutdown operation, e.g. delete ctrls have finished */
2953 flush_workqueue(nvmet_wq);
2955 /* sanity check - all lports should be removed */
2956 if (!list_empty(&nvmet_fc_target_list))
2957 pr_warn("%s: targetport list not empty\n", __func__);
2959 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2961 ida_destroy(&nvmet_fc_tgtport_cnt);
2964 module_init(nvmet_fc_init_module);
2965 module_exit(nvmet_fc_exit_module);
2967 MODULE_DESCRIPTION("NVMe target FC transport driver");
2968 MODULE_LICENSE("GPL v2");