1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016 Avago Technologies. All rights reserved.
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/parser.h>
10 #include <linux/random.h>
11 #include <uapi/scsi/fc/fc_fs.h>
12 #include <uapi/scsi/fc/fc_els.h>
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "../host/fc.h"
20 /* *************************** Data Structures/Defines ****************** */
23 #define NVMET_LS_CTX_COUNT 256
25 struct nvmet_fc_tgtport;
26 struct nvmet_fc_tgt_assoc;
28 struct nvmet_fc_ls_iod { /* for an LS RQST RCV */
29 struct nvmefc_ls_rsp *lsrsp;
30 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
32 struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */
34 struct nvmet_fc_tgtport *tgtport;
35 struct nvmet_fc_tgt_assoc *assoc;
38 union nvmefc_ls_requests *rqstbuf;
39 union nvmefc_ls_responses *rspbuf;
43 struct scatterlist sg[2];
45 struct work_struct work;
46 } __aligned(sizeof(unsigned long long));
48 struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */
49 struct nvmefc_ls_req ls_req;
51 struct nvmet_fc_tgtport *tgtport;
55 struct list_head lsreq_list; /* tgtport->ls_req_list */
60 /* desired maximum for a single sequence - if sg list allows it */
61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
63 enum nvmet_fcp_datadir {
70 struct nvmet_fc_fcp_iod {
71 struct nvmefc_tgt_fcp_req *fcpreq;
73 struct nvme_fc_cmd_iu cmdiubuf;
74 struct nvme_fc_ersp_iu rspiubuf;
76 struct scatterlist *next_sg;
77 struct scatterlist *data_sg;
80 enum nvmet_fcp_datadir io_dir;
88 struct work_struct defer_work;
90 struct nvmet_fc_tgtport *tgtport;
91 struct nvmet_fc_tgt_queue *queue;
93 struct list_head fcp_list; /* tgtport->fcp_list */
96 struct nvmet_fc_tgtport {
97 struct nvmet_fc_target_port fc_target_port;
99 struct list_head tgt_list; /* nvmet_fc_target_list */
100 struct device *dev; /* dev for dma mapping */
101 struct nvmet_fc_target_template *ops;
103 struct nvmet_fc_ls_iod *iod;
105 struct list_head ls_rcv_list;
106 struct list_head ls_req_list;
107 struct list_head ls_busylist;
108 struct list_head assoc_list;
109 struct list_head host_list;
110 struct ida assoc_cnt;
111 struct nvmet_fc_port_entry *pe;
116 struct nvmet_fc_port_entry {
117 struct nvmet_fc_tgtport *tgtport;
118 struct nvmet_port *port;
121 struct list_head pe_list;
124 struct nvmet_fc_defer_fcp_req {
125 struct list_head req_list;
126 struct nvmefc_tgt_fcp_req *fcp_req;
129 struct nvmet_fc_tgt_queue {
140 struct nvmet_cq nvme_cq;
141 struct nvmet_sq nvme_sq;
142 struct nvmet_fc_tgt_assoc *assoc;
143 struct list_head fod_list;
144 struct list_head pending_cmd_list;
145 struct list_head avail_defer_list;
146 struct workqueue_struct *work_q;
148 struct nvmet_fc_fcp_iod fod[]; /* array of fcp_iods */
149 } __aligned(sizeof(unsigned long long));
151 struct nvmet_fc_hostport {
152 struct nvmet_fc_tgtport *tgtport;
154 struct list_head host_list;
159 struct nvmet_fc_tgt_assoc {
162 atomic_t terminating;
163 struct nvmet_fc_tgtport *tgtport;
164 struct nvmet_fc_hostport *hostport;
165 struct nvmet_fc_ls_iod *rcv_disconn;
166 struct list_head a_list;
167 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
169 struct work_struct del_work;
174 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
176 return (iodptr - iodptr->tgtport->iod);
180 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
182 return (fodptr - fodptr->queue->fod);
187 * Association and Connection IDs:
189 * Association ID will have random number in upper 6 bytes and zero
192 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
194 * note: Association ID = Connection ID for queue 0
196 #define BYTES_FOR_QID sizeof(u16)
197 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
198 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
201 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
203 return (assoc->association_id | qid);
207 nvmet_fc_getassociationid(u64 connectionid)
209 return connectionid & ~NVMET_FC_QUEUEID_MASK;
213 nvmet_fc_getqueueid(u64 connectionid)
215 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
218 static inline struct nvmet_fc_tgtport *
219 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
221 return container_of(targetport, struct nvmet_fc_tgtport,
225 static inline struct nvmet_fc_fcp_iod *
226 nvmet_req_to_fod(struct nvmet_req *nvme_req)
228 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
232 /* *************************** Globals **************************** */
235 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
237 static LIST_HEAD(nvmet_fc_target_list);
238 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
239 static LIST_HEAD(nvmet_fc_portentry_list);
242 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
243 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
244 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
245 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
246 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
247 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
248 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
249 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
250 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
251 struct nvmet_fc_fcp_iod *fod);
252 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
253 static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
254 struct nvmet_fc_ls_iod *iod);
257 /* *********************** FC-NVME DMA Handling **************************** */
260 * The fcloop device passes in a NULL device pointer. Real LLD's will
261 * pass in a valid device pointer. If NULL is passed to the dma mapping
262 * routines, depending on the platform, it may or may not succeed, and
266 * Wrapper all the dma routines and check the dev pointer.
268 * If simple mappings (return just a dma address, we'll noop them,
269 * returning a dma address of 0.
271 * On more complex mappings (dma_map_sg), a pseudo routine fills
272 * in the scatter list, setting all dma addresses to 0.
275 static inline dma_addr_t
276 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
277 enum dma_data_direction dir)
279 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
283 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
285 return dev ? dma_mapping_error(dev, dma_addr) : 0;
289 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
290 enum dma_data_direction dir)
293 dma_unmap_single(dev, addr, size, dir);
297 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
298 enum dma_data_direction dir)
301 dma_sync_single_for_cpu(dev, addr, size, dir);
305 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
306 enum dma_data_direction dir)
309 dma_sync_single_for_device(dev, addr, size, dir);
312 /* pseudo dma_map_sg call */
314 fc_map_sg(struct scatterlist *sg, int nents)
316 struct scatterlist *s;
319 WARN_ON(nents == 0 || sg[0].length == 0);
321 for_each_sg(sg, s, nents, i) {
323 #ifdef CONFIG_NEED_SG_DMA_LENGTH
324 s->dma_length = s->length;
331 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
332 enum dma_data_direction dir)
334 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
338 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
339 enum dma_data_direction dir)
342 dma_unmap_sg(dev, sg, nents, dir);
346 /* ********************** FC-NVME LS XMT Handling ************************* */
350 __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)
352 struct nvmet_fc_tgtport *tgtport = lsop->tgtport;
353 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
356 spin_lock_irqsave(&tgtport->lock, flags);
358 if (!lsop->req_queued) {
359 spin_unlock_irqrestore(&tgtport->lock, flags);
363 list_del(&lsop->lsreq_list);
365 lsop->req_queued = false;
367 spin_unlock_irqrestore(&tgtport->lock, flags);
369 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
370 (lsreq->rqstlen + lsreq->rsplen),
374 nvmet_fc_tgtport_put(tgtport);
378 __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,
379 struct nvmet_fc_ls_req_op *lsop,
380 void (*done)(struct nvmefc_ls_req *req, int status))
382 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
386 if (!tgtport->ops->ls_req)
389 if (!nvmet_fc_tgtport_get(tgtport))
393 lsop->req_queued = false;
394 INIT_LIST_HEAD(&lsop->lsreq_list);
396 lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr,
397 lsreq->rqstlen + lsreq->rsplen,
399 if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) {
403 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
405 spin_lock_irqsave(&tgtport->lock, flags);
407 list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list);
409 lsop->req_queued = true;
411 spin_unlock_irqrestore(&tgtport->lock, flags);
413 ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle,
421 lsop->ls_error = ret;
422 spin_lock_irqsave(&tgtport->lock, flags);
423 lsop->req_queued = false;
424 list_del(&lsop->lsreq_list);
425 spin_unlock_irqrestore(&tgtport->lock, flags);
426 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
427 (lsreq->rqstlen + lsreq->rsplen),
430 nvmet_fc_tgtport_put(tgtport);
436 nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport,
437 struct nvmet_fc_ls_req_op *lsop,
438 void (*done)(struct nvmefc_ls_req *req, int status))
440 /* don't wait for completion */
442 return __nvmet_fc_send_ls_req(tgtport, lsop, done);
446 nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
448 struct nvmet_fc_ls_req_op *lsop =
449 container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req);
451 __nvmet_fc_finish_ls_req(lsop);
453 /* fc-nvme target doesn't care about success or failure of cmd */
459 * This routine sends a FC-NVME LS to disconnect (aka terminate)
460 * the FC-NVME Association. Terminating the association also
461 * terminates the FC-NVME connections (per queue, both admin and io
462 * queues) that are part of the association. E.g. things are torn
463 * down, and the related FC-NVME Association ID and Connection IDs
466 * The behavior of the fc-nvme target is such that it's
467 * understanding of the association and connections will implicitly
468 * be torn down. The action is implicit as it may be due to a loss of
469 * connectivity with the fc-nvme host, so the target may never get a
470 * response even if it tried. As such, the action of this routine
471 * is to asynchronously send the LS, ignore any results of the LS, and
472 * continue on with terminating the association. If the fc-nvme host
473 * is present and receives the LS, it too can tear down.
476 nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)
478 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
479 struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
480 struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
481 struct nvmet_fc_ls_req_op *lsop;
482 struct nvmefc_ls_req *lsreq;
486 * If ls_req is NULL or no hosthandle, it's an older lldd and no
487 * message is normal. Otherwise, send unless the hostport has
488 * already been invalidated by the lldd.
490 if (!tgtport->ops->ls_req || !assoc->hostport ||
491 assoc->hostport->invalid)
494 lsop = kzalloc((sizeof(*lsop) +
495 sizeof(*discon_rqst) + sizeof(*discon_acc) +
496 tgtport->ops->lsrqst_priv_sz), GFP_KERNEL);
498 dev_info(tgtport->dev,
499 "{%d:%d} send Disconnect Association failed: ENOMEM\n",
500 tgtport->fc_target_port.port_num, assoc->a_id);
504 discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
505 discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
506 lsreq = &lsop->ls_req;
507 if (tgtport->ops->lsrqst_priv_sz)
508 lsreq->private = (void *)&discon_acc[1];
510 lsreq->private = NULL;
512 lsop->tgtport = tgtport;
513 lsop->hosthandle = assoc->hostport->hosthandle;
515 nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
516 assoc->association_id);
518 ret = nvmet_fc_send_ls_req_async(tgtport, lsop,
519 nvmet_fc_disconnect_assoc_done);
521 dev_info(tgtport->dev,
522 "{%d:%d} XMT Disconnect Association failed: %d\n",
523 tgtport->fc_target_port.port_num, assoc->a_id, ret);
529 /* *********************** FC-NVME Port Management ************************ */
533 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
535 struct nvmet_fc_ls_iod *iod;
538 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
545 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
546 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
547 iod->tgtport = tgtport;
548 list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
550 iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) +
551 sizeof(union nvmefc_ls_responses),
556 iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1];
558 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
559 sizeof(*iod->rspbuf),
561 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
569 list_del(&iod->ls_rcv_list);
570 for (iod--, i--; i >= 0; iod--, i--) {
571 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
572 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
574 list_del(&iod->ls_rcv_list);
583 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
585 struct nvmet_fc_ls_iod *iod = tgtport->iod;
588 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
589 fc_dma_unmap_single(tgtport->dev,
590 iod->rspdma, sizeof(*iod->rspbuf),
593 list_del(&iod->ls_rcv_list);
598 static struct nvmet_fc_ls_iod *
599 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
601 struct nvmet_fc_ls_iod *iod;
604 spin_lock_irqsave(&tgtport->lock, flags);
605 iod = list_first_entry_or_null(&tgtport->ls_rcv_list,
606 struct nvmet_fc_ls_iod, ls_rcv_list);
608 list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist);
609 spin_unlock_irqrestore(&tgtport->lock, flags);
615 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
616 struct nvmet_fc_ls_iod *iod)
620 spin_lock_irqsave(&tgtport->lock, flags);
621 list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
622 spin_unlock_irqrestore(&tgtport->lock, flags);
626 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
627 struct nvmet_fc_tgt_queue *queue)
629 struct nvmet_fc_fcp_iod *fod = queue->fod;
632 for (i = 0; i < queue->sqsize; fod++, i++) {
633 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
634 fod->tgtport = tgtport;
638 fod->aborted = false;
640 list_add_tail(&fod->fcp_list, &queue->fod_list);
641 spin_lock_init(&fod->flock);
643 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
644 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
645 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
646 list_del(&fod->fcp_list);
647 for (fod--, i--; i >= 0; fod--, i--) {
648 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
649 sizeof(fod->rspiubuf),
652 list_del(&fod->fcp_list);
661 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
662 struct nvmet_fc_tgt_queue *queue)
664 struct nvmet_fc_fcp_iod *fod = queue->fod;
667 for (i = 0; i < queue->sqsize; fod++, i++) {
669 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
670 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
674 static struct nvmet_fc_fcp_iod *
675 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
677 struct nvmet_fc_fcp_iod *fod;
679 lockdep_assert_held(&queue->qlock);
681 fod = list_first_entry_or_null(&queue->fod_list,
682 struct nvmet_fc_fcp_iod, fcp_list);
684 list_del(&fod->fcp_list);
687 * no queue reference is taken, as it was taken by the
688 * queue lookup just prior to the allocation. The iod
689 * will "inherit" that reference.
697 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
698 struct nvmet_fc_tgt_queue *queue,
699 struct nvmefc_tgt_fcp_req *fcpreq)
701 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
704 * put all admin cmds on hw queue id 0. All io commands go to
705 * the respective hw queue based on a modulo basis
707 fcpreq->hwqid = queue->qid ?
708 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
710 nvmet_fc_handle_fcp_rqst(tgtport, fod);
714 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
716 struct nvmet_fc_fcp_iod *fod =
717 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
719 /* Submit deferred IO for processing */
720 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
725 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
726 struct nvmet_fc_fcp_iod *fod)
728 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
729 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
730 struct nvmet_fc_defer_fcp_req *deferfcp;
733 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
734 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
736 fcpreq->nvmet_fc_private = NULL;
740 fod->aborted = false;
741 fod->writedataactive = false;
744 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
746 /* release the queue lookup reference on the completed IO */
747 nvmet_fc_tgt_q_put(queue);
749 spin_lock_irqsave(&queue->qlock, flags);
750 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
751 struct nvmet_fc_defer_fcp_req, req_list);
753 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
754 spin_unlock_irqrestore(&queue->qlock, flags);
758 /* Re-use the fod for the next pending cmd that was deferred */
759 list_del(&deferfcp->req_list);
761 fcpreq = deferfcp->fcp_req;
763 /* deferfcp can be reused for another IO at a later date */
764 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
766 spin_unlock_irqrestore(&queue->qlock, flags);
768 /* Save NVME CMD IO in fod */
769 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
771 /* Setup new fcpreq to be processed */
772 fcpreq->rspaddr = NULL;
774 fcpreq->nvmet_fc_private = fod;
775 fod->fcpreq = fcpreq;
778 /* inform LLDD IO is now being processed */
779 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
782 * Leave the queue lookup get reference taken when
783 * fod was originally allocated.
786 queue_work(queue->work_q, &fod->defer_work);
789 static struct nvmet_fc_tgt_queue *
790 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
793 struct nvmet_fc_tgt_queue *queue;
797 if (qid > NVMET_NR_QUEUES)
800 queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL);
804 if (!nvmet_fc_tgt_a_get(assoc))
807 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
808 assoc->tgtport->fc_target_port.port_num,
814 queue->sqsize = sqsize;
815 queue->assoc = assoc;
816 INIT_LIST_HEAD(&queue->fod_list);
817 INIT_LIST_HEAD(&queue->avail_defer_list);
818 INIT_LIST_HEAD(&queue->pending_cmd_list);
819 atomic_set(&queue->connected, 0);
820 atomic_set(&queue->sqtail, 0);
821 atomic_set(&queue->rsn, 1);
822 atomic_set(&queue->zrspcnt, 0);
823 spin_lock_init(&queue->qlock);
824 kref_init(&queue->ref);
826 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
828 ret = nvmet_sq_init(&queue->nvme_sq);
830 goto out_fail_iodlist;
832 WARN_ON(assoc->queues[qid]);
833 spin_lock_irqsave(&assoc->tgtport->lock, flags);
834 assoc->queues[qid] = queue;
835 spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
840 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
841 destroy_workqueue(queue->work_q);
843 nvmet_fc_tgt_a_put(assoc);
851 nvmet_fc_tgt_queue_free(struct kref *ref)
853 struct nvmet_fc_tgt_queue *queue =
854 container_of(ref, struct nvmet_fc_tgt_queue, ref);
857 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
858 queue->assoc->queues[queue->qid] = NULL;
859 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
861 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
863 nvmet_fc_tgt_a_put(queue->assoc);
865 destroy_workqueue(queue->work_q);
871 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
873 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
877 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
879 return kref_get_unless_zero(&queue->ref);
884 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
886 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
887 struct nvmet_fc_fcp_iod *fod = queue->fod;
888 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
893 disconnect = atomic_xchg(&queue->connected, 0);
895 /* if not connected, nothing to do */
899 spin_lock_irqsave(&queue->qlock, flags);
900 /* abort outstanding io's */
901 for (i = 0; i < queue->sqsize; fod++, i++) {
903 spin_lock(&fod->flock);
906 * only call lldd abort routine if waiting for
907 * writedata. other outstanding ops should finish
910 if (fod->writedataactive) {
912 spin_unlock(&fod->flock);
913 tgtport->ops->fcp_abort(
914 &tgtport->fc_target_port, fod->fcpreq);
916 spin_unlock(&fod->flock);
920 /* Cleanup defer'ed IOs in queue */
921 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
923 list_del(&deferfcp->req_list);
928 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
929 struct nvmet_fc_defer_fcp_req, req_list);
933 list_del(&deferfcp->req_list);
934 spin_unlock_irqrestore(&queue->qlock, flags);
936 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
939 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
942 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
945 /* release the queue lookup reference */
946 nvmet_fc_tgt_q_put(queue);
950 spin_lock_irqsave(&queue->qlock, flags);
952 spin_unlock_irqrestore(&queue->qlock, flags);
954 flush_workqueue(queue->work_q);
956 nvmet_sq_destroy(&queue->nvme_sq);
958 nvmet_fc_tgt_q_put(queue);
961 static struct nvmet_fc_tgt_queue *
962 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
965 struct nvmet_fc_tgt_assoc *assoc;
966 struct nvmet_fc_tgt_queue *queue;
967 u64 association_id = nvmet_fc_getassociationid(connection_id);
968 u16 qid = nvmet_fc_getqueueid(connection_id);
971 if (qid > NVMET_NR_QUEUES)
974 spin_lock_irqsave(&tgtport->lock, flags);
975 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
976 if (association_id == assoc->association_id) {
977 queue = assoc->queues[qid];
979 (!atomic_read(&queue->connected) ||
980 !nvmet_fc_tgt_q_get(queue)))
982 spin_unlock_irqrestore(&tgtport->lock, flags);
986 spin_unlock_irqrestore(&tgtport->lock, flags);
991 nvmet_fc_hostport_free(struct kref *ref)
993 struct nvmet_fc_hostport *hostport =
994 container_of(ref, struct nvmet_fc_hostport, ref);
995 struct nvmet_fc_tgtport *tgtport = hostport->tgtport;
998 spin_lock_irqsave(&tgtport->lock, flags);
999 list_del(&hostport->host_list);
1000 spin_unlock_irqrestore(&tgtport->lock, flags);
1001 if (tgtport->ops->host_release && hostport->invalid)
1002 tgtport->ops->host_release(hostport->hosthandle);
1004 nvmet_fc_tgtport_put(tgtport);
1008 nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport)
1010 kref_put(&hostport->ref, nvmet_fc_hostport_free);
1014 nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport)
1016 return kref_get_unless_zero(&hostport->ref);
1020 nvmet_fc_free_hostport(struct nvmet_fc_hostport *hostport)
1022 /* if LLDD not implemented, leave as NULL */
1023 if (!hostport || !hostport->hosthandle)
1026 nvmet_fc_hostport_put(hostport);
1029 static struct nvmet_fc_hostport *
1030 nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1032 struct nvmet_fc_hostport *newhost, *host, *match = NULL;
1033 unsigned long flags;
1035 /* if LLDD not implemented, leave as NULL */
1039 /* take reference for what will be the newly allocated hostport */
1040 if (!nvmet_fc_tgtport_get(tgtport))
1041 return ERR_PTR(-EINVAL);
1043 newhost = kzalloc(sizeof(*newhost), GFP_KERNEL);
1045 spin_lock_irqsave(&tgtport->lock, flags);
1046 list_for_each_entry(host, &tgtport->host_list, host_list) {
1047 if (host->hosthandle == hosthandle && !host->invalid) {
1048 if (nvmet_fc_hostport_get(host)) {
1054 spin_unlock_irqrestore(&tgtport->lock, flags);
1055 /* no allocation - release reference */
1056 nvmet_fc_tgtport_put(tgtport);
1057 return (match) ? match : ERR_PTR(-ENOMEM);
1060 newhost->tgtport = tgtport;
1061 newhost->hosthandle = hosthandle;
1062 INIT_LIST_HEAD(&newhost->host_list);
1063 kref_init(&newhost->ref);
1065 spin_lock_irqsave(&tgtport->lock, flags);
1066 list_for_each_entry(host, &tgtport->host_list, host_list) {
1067 if (host->hosthandle == hosthandle && !host->invalid) {
1068 if (nvmet_fc_hostport_get(host)) {
1077 /* releasing allocation - release reference */
1078 nvmet_fc_tgtport_put(tgtport);
1080 list_add_tail(&newhost->host_list, &tgtport->host_list);
1081 spin_unlock_irqrestore(&tgtport->lock, flags);
1083 return (match) ? match : newhost;
1087 nvmet_fc_delete_assoc(struct work_struct *work)
1089 struct nvmet_fc_tgt_assoc *assoc =
1090 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
1092 nvmet_fc_delete_target_assoc(assoc);
1093 nvmet_fc_tgt_a_put(assoc);
1096 static struct nvmet_fc_tgt_assoc *
1097 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1099 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
1100 unsigned long flags;
1103 bool needrandom = true;
1108 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
1112 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
1114 goto out_free_assoc;
1116 if (!nvmet_fc_tgtport_get(tgtport))
1119 assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle);
1120 if (IS_ERR(assoc->hostport))
1123 assoc->tgtport = tgtport;
1125 INIT_LIST_HEAD(&assoc->a_list);
1126 kref_init(&assoc->ref);
1127 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc);
1128 atomic_set(&assoc->terminating, 0);
1130 while (needrandom) {
1131 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
1132 ran = ran << BYTES_FOR_QID_SHIFT;
1134 spin_lock_irqsave(&tgtport->lock, flags);
1136 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) {
1137 if (ran == tmpassoc->association_id) {
1143 assoc->association_id = ran;
1144 list_add_tail(&assoc->a_list, &tgtport->assoc_list);
1146 spin_unlock_irqrestore(&tgtport->lock, flags);
1152 nvmet_fc_tgtport_put(tgtport);
1154 ida_simple_remove(&tgtport->assoc_cnt, idx);
1161 nvmet_fc_target_assoc_free(struct kref *ref)
1163 struct nvmet_fc_tgt_assoc *assoc =
1164 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
1165 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1166 struct nvmet_fc_ls_iod *oldls;
1167 unsigned long flags;
1169 /* Send Disconnect now that all i/o has completed */
1170 nvmet_fc_xmt_disconnect_assoc(assoc);
1172 nvmet_fc_free_hostport(assoc->hostport);
1173 spin_lock_irqsave(&tgtport->lock, flags);
1174 list_del(&assoc->a_list);
1175 oldls = assoc->rcv_disconn;
1176 spin_unlock_irqrestore(&tgtport->lock, flags);
1177 /* if pending Rcv Disconnect Association LS, send rsp now */
1179 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1180 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
1181 dev_info(tgtport->dev,
1182 "{%d:%d} Association freed\n",
1183 tgtport->fc_target_port.port_num, assoc->a_id);
1185 nvmet_fc_tgtport_put(tgtport);
1189 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
1191 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
1195 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
1197 return kref_get_unless_zero(&assoc->ref);
1201 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
1203 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1204 struct nvmet_fc_tgt_queue *queue;
1205 unsigned long flags;
1208 terminating = atomic_xchg(&assoc->terminating, 1);
1210 /* if already terminating, do nothing */
1214 spin_lock_irqsave(&tgtport->lock, flags);
1215 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1216 queue = assoc->queues[i];
1218 if (!nvmet_fc_tgt_q_get(queue))
1220 spin_unlock_irqrestore(&tgtport->lock, flags);
1221 nvmet_fc_delete_target_queue(queue);
1222 nvmet_fc_tgt_q_put(queue);
1223 spin_lock_irqsave(&tgtport->lock, flags);
1226 spin_unlock_irqrestore(&tgtport->lock, flags);
1228 dev_info(tgtport->dev,
1229 "{%d:%d} Association deleted\n",
1230 tgtport->fc_target_port.port_num, assoc->a_id);
1232 nvmet_fc_tgt_a_put(assoc);
1235 static struct nvmet_fc_tgt_assoc *
1236 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
1239 struct nvmet_fc_tgt_assoc *assoc;
1240 struct nvmet_fc_tgt_assoc *ret = NULL;
1241 unsigned long flags;
1243 spin_lock_irqsave(&tgtport->lock, flags);
1244 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1245 if (association_id == assoc->association_id) {
1247 if (!nvmet_fc_tgt_a_get(assoc))
1252 spin_unlock_irqrestore(&tgtport->lock, flags);
1258 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport,
1259 struct nvmet_fc_port_entry *pe,
1260 struct nvmet_port *port)
1262 lockdep_assert_held(&nvmet_fc_tgtlock);
1264 pe->tgtport = tgtport;
1270 pe->node_name = tgtport->fc_target_port.node_name;
1271 pe->port_name = tgtport->fc_target_port.port_name;
1272 INIT_LIST_HEAD(&pe->pe_list);
1274 list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list);
1278 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe)
1280 unsigned long flags;
1282 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1284 pe->tgtport->pe = NULL;
1285 list_del(&pe->pe_list);
1286 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1290 * called when a targetport deregisters. Breaks the relationship
1291 * with the nvmet port, but leaves the port_entry in place so that
1292 * re-registration can resume operation.
1295 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport)
1297 struct nvmet_fc_port_entry *pe;
1298 unsigned long flags;
1300 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1305 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1309 * called when a new targetport is registered. Looks in the
1310 * existing nvmet port_entries to see if the nvmet layer is
1311 * configured for the targetport's wwn's. (the targetport existed,
1312 * nvmet configured, the lldd unregistered the tgtport, and is now
1313 * reregistering the same targetport). If so, set the nvmet port
1314 * port entry on the targetport.
1317 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport)
1319 struct nvmet_fc_port_entry *pe;
1320 unsigned long flags;
1322 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1323 list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) {
1324 if (tgtport->fc_target_port.node_name == pe->node_name &&
1325 tgtport->fc_target_port.port_name == pe->port_name) {
1326 WARN_ON(pe->tgtport);
1328 pe->tgtport = tgtport;
1332 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1336 * nvme_fc_register_targetport - transport entry point called by an
1337 * LLDD to register the existence of a local
1338 * NVME subystem FC port.
1339 * @pinfo: pointer to information about the port to be registered
1340 * @template: LLDD entrypoints and operational parameters for the port
1341 * @dev: physical hardware device node port corresponds to. Will be
1342 * used for DMA mappings
1343 * @portptr: pointer to a local port pointer. Upon success, the routine
1344 * will allocate a nvme_fc_local_port structure and place its
1345 * address in the local port pointer. Upon failure, local port
1346 * pointer will be set to NULL.
1349 * a completion status. Must be 0 upon success; a negative errno
1350 * (ex: -ENXIO) upon failure.
1353 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
1354 struct nvmet_fc_target_template *template,
1356 struct nvmet_fc_target_port **portptr)
1358 struct nvmet_fc_tgtport *newrec;
1359 unsigned long flags;
1362 if (!template->xmt_ls_rsp || !template->fcp_op ||
1363 !template->fcp_abort ||
1364 !template->fcp_req_release || !template->targetport_delete ||
1365 !template->max_hw_queues || !template->max_sgl_segments ||
1366 !template->max_dif_sgl_segments || !template->dma_boundary) {
1368 goto out_regtgt_failed;
1371 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
1375 goto out_regtgt_failed;
1378 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
1381 goto out_fail_kfree;
1384 if (!get_device(dev) && dev) {
1389 newrec->fc_target_port.node_name = pinfo->node_name;
1390 newrec->fc_target_port.port_name = pinfo->port_name;
1391 if (template->target_priv_sz)
1392 newrec->fc_target_port.private = &newrec[1];
1394 newrec->fc_target_port.private = NULL;
1395 newrec->fc_target_port.port_id = pinfo->port_id;
1396 newrec->fc_target_port.port_num = idx;
1397 INIT_LIST_HEAD(&newrec->tgt_list);
1399 newrec->ops = template;
1400 spin_lock_init(&newrec->lock);
1401 INIT_LIST_HEAD(&newrec->ls_rcv_list);
1402 INIT_LIST_HEAD(&newrec->ls_req_list);
1403 INIT_LIST_HEAD(&newrec->ls_busylist);
1404 INIT_LIST_HEAD(&newrec->assoc_list);
1405 INIT_LIST_HEAD(&newrec->host_list);
1406 kref_init(&newrec->ref);
1407 ida_init(&newrec->assoc_cnt);
1408 newrec->max_sg_cnt = template->max_sgl_segments;
1410 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1413 goto out_free_newrec;
1416 nvmet_fc_portentry_rebind_tgt(newrec);
1418 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1419 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1420 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1422 *portptr = &newrec->fc_target_port;
1428 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1435 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1439 nvmet_fc_free_tgtport(struct kref *ref)
1441 struct nvmet_fc_tgtport *tgtport =
1442 container_of(ref, struct nvmet_fc_tgtport, ref);
1443 struct device *dev = tgtport->dev;
1444 unsigned long flags;
1446 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1447 list_del(&tgtport->tgt_list);
1448 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1450 nvmet_fc_free_ls_iodlist(tgtport);
1452 /* let the LLDD know we've finished tearing it down */
1453 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1455 ida_simple_remove(&nvmet_fc_tgtport_cnt,
1456 tgtport->fc_target_port.port_num);
1458 ida_destroy(&tgtport->assoc_cnt);
1466 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1468 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1472 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1474 return kref_get_unless_zero(&tgtport->ref);
1478 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1480 struct nvmet_fc_tgt_assoc *assoc, *next;
1481 unsigned long flags;
1483 spin_lock_irqsave(&tgtport->lock, flags);
1484 list_for_each_entry_safe(assoc, next,
1485 &tgtport->assoc_list, a_list) {
1486 if (!nvmet_fc_tgt_a_get(assoc))
1488 if (!schedule_work(&assoc->del_work))
1489 /* already deleting - release local reference */
1490 nvmet_fc_tgt_a_put(assoc);
1492 spin_unlock_irqrestore(&tgtport->lock, flags);
1496 * nvmet_fc_invalidate_host - transport entry point called by an LLDD
1497 * to remove references to a hosthandle for LS's.
1499 * The nvmet-fc layer ensures that any references to the hosthandle
1500 * on the targetport are forgotten (set to NULL). The LLDD will
1501 * typically call this when a login with a remote host port has been
1502 * lost, thus LS's for the remote host port are no longer possible.
1504 * If an LS request is outstanding to the targetport/hosthandle (or
1505 * issued concurrently with the call to invalidate the host), the
1506 * LLDD is responsible for terminating/aborting the LS and completing
1507 * the LS request. It is recommended that these terminations/aborts
1508 * occur after calling to invalidate the host handle to avoid additional
1509 * retries by the nvmet-fc transport. The nvmet-fc transport may
1510 * continue to reference host handle while it cleans up outstanding
1511 * NVME associations. The nvmet-fc transport will call the
1512 * ops->host_release() callback to notify the LLDD that all references
1513 * are complete and the related host handle can be recovered.
1514 * Note: if there are no references, the callback may be called before
1515 * the invalidate host call returns.
1517 * @target_port: pointer to the (registered) target port that a prior
1518 * LS was received on and which supplied the transport the
1520 * @hosthandle: the handle (pointer) that represents the host port
1521 * that no longer has connectivity and that LS's should
1522 * no longer be directed to.
1525 nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,
1528 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1529 struct nvmet_fc_tgt_assoc *assoc, *next;
1530 unsigned long flags;
1531 bool noassoc = true;
1533 spin_lock_irqsave(&tgtport->lock, flags);
1534 list_for_each_entry_safe(assoc, next,
1535 &tgtport->assoc_list, a_list) {
1536 if (!assoc->hostport ||
1537 assoc->hostport->hosthandle != hosthandle)
1539 if (!nvmet_fc_tgt_a_get(assoc))
1541 assoc->hostport->invalid = 1;
1543 if (!schedule_work(&assoc->del_work))
1544 /* already deleting - release local reference */
1545 nvmet_fc_tgt_a_put(assoc);
1547 spin_unlock_irqrestore(&tgtport->lock, flags);
1549 /* if there's nothing to wait for - call the callback */
1550 if (noassoc && tgtport->ops->host_release)
1551 tgtport->ops->host_release(hosthandle);
1553 EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host);
1556 * nvmet layer has called to terminate an association
1559 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1561 struct nvmet_fc_tgtport *tgtport, *next;
1562 struct nvmet_fc_tgt_assoc *assoc;
1563 struct nvmet_fc_tgt_queue *queue;
1564 unsigned long flags;
1565 bool found_ctrl = false;
1567 /* this is a bit ugly, but don't want to make locks layered */
1568 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1569 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1571 if (!nvmet_fc_tgtport_get(tgtport))
1573 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1575 spin_lock_irqsave(&tgtport->lock, flags);
1576 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1577 queue = assoc->queues[0];
1578 if (queue && queue->nvme_sq.ctrl == ctrl) {
1579 if (nvmet_fc_tgt_a_get(assoc))
1584 spin_unlock_irqrestore(&tgtport->lock, flags);
1586 nvmet_fc_tgtport_put(tgtport);
1589 if (!schedule_work(&assoc->del_work))
1590 /* already deleting - release local reference */
1591 nvmet_fc_tgt_a_put(assoc);
1595 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1597 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1601 * nvme_fc_unregister_targetport - transport entry point called by an
1602 * LLDD to deregister/remove a previously
1603 * registered a local NVME subsystem FC port.
1604 * @target_port: pointer to the (registered) target port that is to be
1608 * a completion status. Must be 0 upon success; a negative errno
1609 * (ex: -ENXIO) upon failure.
1612 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1614 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1616 nvmet_fc_portentry_unbind_tgt(tgtport);
1618 /* terminate any outstanding associations */
1619 __nvmet_fc_free_assocs(tgtport);
1622 * should terminate LS's as well. However, LS's will be generated
1623 * at the tail end of association termination, so they likely don't
1624 * exist yet. And even if they did, it's worthwhile to just let
1625 * them finish and targetport ref counting will clean things up.
1628 nvmet_fc_tgtport_put(tgtport);
1632 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1635 /* ********************** FC-NVME LS RCV Handling ************************* */
1639 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1640 struct nvmet_fc_ls_iod *iod)
1642 struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc;
1643 struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc;
1644 struct nvmet_fc_tgt_queue *queue;
1647 memset(acc, 0, sizeof(*acc));
1650 * FC-NVME spec changes. There are initiators sending different
1651 * lengths as padding sizes for Create Association Cmd descriptor
1653 * Accept anything of "minimum" length. Assume format per 1.15
1654 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1655 * trailing pad length is.
1657 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1658 ret = VERR_CR_ASSOC_LEN;
1659 else if (be32_to_cpu(rqst->desc_list_len) <
1660 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1661 ret = VERR_CR_ASSOC_RQST_LEN;
1662 else if (rqst->assoc_cmd.desc_tag !=
1663 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1664 ret = VERR_CR_ASSOC_CMD;
1665 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1666 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1667 ret = VERR_CR_ASSOC_CMD_LEN;
1668 else if (!rqst->assoc_cmd.ersp_ratio ||
1669 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1670 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1671 ret = VERR_ERSP_RATIO;
1674 /* new association w/ admin queue */
1675 iod->assoc = nvmet_fc_alloc_target_assoc(
1676 tgtport, iod->hosthandle);
1678 ret = VERR_ASSOC_ALLOC_FAIL;
1680 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1681 be16_to_cpu(rqst->assoc_cmd.sqsize));
1683 ret = VERR_QUEUE_ALLOC_FAIL;
1684 nvmet_fc_tgt_a_put(iod->assoc);
1690 dev_err(tgtport->dev,
1691 "Create Association LS failed: %s\n",
1692 validation_errors[ret]);
1693 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1694 sizeof(*acc), rqst->w0.ls_cmd,
1695 FCNVME_RJT_RC_LOGIC,
1696 FCNVME_RJT_EXP_NONE, 0);
1700 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1701 atomic_set(&queue->connected, 1);
1702 queue->sqhd = 0; /* best place to init value */
1704 dev_info(tgtport->dev,
1705 "{%d:%d} Association created\n",
1706 tgtport->fc_target_port.port_num, iod->assoc->a_id);
1708 /* format a response */
1710 iod->lsrsp->rsplen = sizeof(*acc);
1712 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1714 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1715 FCNVME_LS_CREATE_ASSOCIATION);
1716 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1717 acc->associd.desc_len =
1719 sizeof(struct fcnvme_lsdesc_assoc_id));
1720 acc->associd.association_id =
1721 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1722 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1723 acc->connectid.desc_len =
1725 sizeof(struct fcnvme_lsdesc_conn_id));
1726 acc->connectid.connection_id = acc->associd.association_id;
1730 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1731 struct nvmet_fc_ls_iod *iod)
1733 struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn;
1734 struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn;
1735 struct nvmet_fc_tgt_queue *queue;
1738 memset(acc, 0, sizeof(*acc));
1740 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1741 ret = VERR_CR_CONN_LEN;
1742 else if (rqst->desc_list_len !=
1744 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1745 ret = VERR_CR_CONN_RQST_LEN;
1746 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1747 ret = VERR_ASSOC_ID;
1748 else if (rqst->associd.desc_len !=
1750 sizeof(struct fcnvme_lsdesc_assoc_id)))
1751 ret = VERR_ASSOC_ID_LEN;
1752 else if (rqst->connect_cmd.desc_tag !=
1753 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1754 ret = VERR_CR_CONN_CMD;
1755 else if (rqst->connect_cmd.desc_len !=
1757 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1758 ret = VERR_CR_CONN_CMD_LEN;
1759 else if (!rqst->connect_cmd.ersp_ratio ||
1760 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1761 be16_to_cpu(rqst->connect_cmd.sqsize)))
1762 ret = VERR_ERSP_RATIO;
1766 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1767 be64_to_cpu(rqst->associd.association_id));
1769 ret = VERR_NO_ASSOC;
1771 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1772 be16_to_cpu(rqst->connect_cmd.qid),
1773 be16_to_cpu(rqst->connect_cmd.sqsize));
1775 ret = VERR_QUEUE_ALLOC_FAIL;
1777 /* release get taken in nvmet_fc_find_target_assoc */
1778 nvmet_fc_tgt_a_put(iod->assoc);
1783 dev_err(tgtport->dev,
1784 "Create Connection LS failed: %s\n",
1785 validation_errors[ret]);
1786 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1787 sizeof(*acc), rqst->w0.ls_cmd,
1788 (ret == VERR_NO_ASSOC) ?
1789 FCNVME_RJT_RC_INV_ASSOC :
1790 FCNVME_RJT_RC_LOGIC,
1791 FCNVME_RJT_EXP_NONE, 0);
1795 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1796 atomic_set(&queue->connected, 1);
1797 queue->sqhd = 0; /* best place to init value */
1799 /* format a response */
1801 iod->lsrsp->rsplen = sizeof(*acc);
1803 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1804 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1805 FCNVME_LS_CREATE_CONNECTION);
1806 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1807 acc->connectid.desc_len =
1809 sizeof(struct fcnvme_lsdesc_conn_id));
1810 acc->connectid.connection_id =
1811 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1812 be16_to_cpu(rqst->connect_cmd.qid)));
1816 * Returns true if the LS response is to be transmit
1817 * Returns false if the LS response is to be delayed
1820 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1821 struct nvmet_fc_ls_iod *iod)
1823 struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1824 &iod->rqstbuf->rq_dis_assoc;
1825 struct fcnvme_ls_disconnect_assoc_acc *acc =
1826 &iod->rspbuf->rsp_dis_assoc;
1827 struct nvmet_fc_tgt_assoc *assoc = NULL;
1828 struct nvmet_fc_ls_iod *oldls = NULL;
1829 unsigned long flags;
1832 memset(acc, 0, sizeof(*acc));
1834 ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst);
1836 /* match an active association - takes an assoc ref if !NULL */
1837 assoc = nvmet_fc_find_target_assoc(tgtport,
1838 be64_to_cpu(rqst->associd.association_id));
1841 ret = VERR_NO_ASSOC;
1844 if (ret || !assoc) {
1845 dev_err(tgtport->dev,
1846 "Disconnect LS failed: %s\n",
1847 validation_errors[ret]);
1848 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1849 sizeof(*acc), rqst->w0.ls_cmd,
1850 (ret == VERR_NO_ASSOC) ?
1851 FCNVME_RJT_RC_INV_ASSOC :
1852 FCNVME_RJT_RC_LOGIC,
1853 FCNVME_RJT_EXP_NONE, 0);
1857 /* format a response */
1859 iod->lsrsp->rsplen = sizeof(*acc);
1861 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1863 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1864 FCNVME_LS_DISCONNECT_ASSOC);
1866 /* release get taken in nvmet_fc_find_target_assoc */
1867 nvmet_fc_tgt_a_put(assoc);
1870 * The rules for LS response says the response cannot
1871 * go back until ABTS's have been sent for all outstanding
1872 * I/O and a Disconnect Association LS has been sent.
1873 * So... save off the Disconnect LS to send the response
1874 * later. If there was a prior LS already saved, replace
1875 * it with the newer one and send a can't perform reject
1878 spin_lock_irqsave(&tgtport->lock, flags);
1879 oldls = assoc->rcv_disconn;
1880 assoc->rcv_disconn = iod;
1881 spin_unlock_irqrestore(&tgtport->lock, flags);
1883 nvmet_fc_delete_target_assoc(assoc);
1886 dev_info(tgtport->dev,
1887 "{%d:%d} Multiple Disconnect Association LS's "
1889 tgtport->fc_target_port.port_num, assoc->a_id);
1890 /* overwrite good response with bogus failure */
1891 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1892 sizeof(*iod->rspbuf),
1893 /* ok to use rqst, LS is same */
1896 FCNVME_RJT_EXP_NONE, 0);
1897 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1904 /* *********************** NVME Ctrl Routines **************************** */
1907 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1909 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1912 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1914 struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private;
1915 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1917 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1918 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1919 nvmet_fc_free_ls_iod(tgtport, iod);
1920 nvmet_fc_tgtport_put(tgtport);
1924 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1925 struct nvmet_fc_ls_iod *iod)
1929 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1930 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1932 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp);
1934 nvmet_fc_xmt_ls_rsp_done(iod->lsrsp);
1938 * Actual processing routine for received FC-NVME LS Requests from the LLD
1941 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1942 struct nvmet_fc_ls_iod *iod)
1944 struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0;
1945 bool sendrsp = true;
1947 iod->lsrsp->nvme_fc_private = iod;
1948 iod->lsrsp->rspbuf = iod->rspbuf;
1949 iod->lsrsp->rspdma = iod->rspdma;
1950 iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done;
1951 /* Be preventative. handlers will later set to valid length */
1952 iod->lsrsp->rsplen = 0;
1958 * parse request input, execute the request, and format the
1961 switch (w0->ls_cmd) {
1962 case FCNVME_LS_CREATE_ASSOCIATION:
1963 /* Creates Association and initial Admin Queue/Connection */
1964 nvmet_fc_ls_create_association(tgtport, iod);
1966 case FCNVME_LS_CREATE_CONNECTION:
1967 /* Creates an IO Queue/Connection */
1968 nvmet_fc_ls_create_connection(tgtport, iod);
1970 case FCNVME_LS_DISCONNECT_ASSOC:
1971 /* Terminate a Queue/Connection or the Association */
1972 sendrsp = nvmet_fc_ls_disconnect(tgtport, iod);
1975 iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf,
1976 sizeof(*iod->rspbuf), w0->ls_cmd,
1977 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1981 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1985 * Actual processing routine for received FC-NVME LS Requests from the LLD
1988 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1990 struct nvmet_fc_ls_iod *iod =
1991 container_of(work, struct nvmet_fc_ls_iod, work);
1992 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1994 nvmet_fc_handle_ls_rqst(tgtport, iod);
1999 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
2000 * upon the reception of a NVME LS request.
2002 * The nvmet-fc layer will copy payload to an internal structure for
2003 * processing. As such, upon completion of the routine, the LLDD may
2004 * immediately free/reuse the LS request buffer passed in the call.
2006 * If this routine returns error, the LLDD should abort the exchange.
2008 * @target_port: pointer to the (registered) target port the LS was
2010 * @lsrsp: pointer to a lsrsp structure to be used to reference
2011 * the exchange corresponding to the LS.
2012 * @lsreqbuf: pointer to the buffer containing the LS Request
2013 * @lsreqbuf_len: length, in bytes, of the received LS request
2016 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
2018 struct nvmefc_ls_rsp *lsrsp,
2019 void *lsreqbuf, u32 lsreqbuf_len)
2021 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2022 struct nvmet_fc_ls_iod *iod;
2023 struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
2025 if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
2026 dev_info(tgtport->dev,
2027 "RCV %s LS failed: payload too large (%d)\n",
2028 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2029 nvmefc_ls_names[w0->ls_cmd] : "",
2034 if (!nvmet_fc_tgtport_get(tgtport)) {
2035 dev_info(tgtport->dev,
2036 "RCV %s LS failed: target deleting\n",
2037 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2038 nvmefc_ls_names[w0->ls_cmd] : "");
2042 iod = nvmet_fc_alloc_ls_iod(tgtport);
2044 dev_info(tgtport->dev,
2045 "RCV %s LS failed: context allocation failed\n",
2046 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2047 nvmefc_ls_names[w0->ls_cmd] : "");
2048 nvmet_fc_tgtport_put(tgtport);
2054 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
2055 iod->rqstdatalen = lsreqbuf_len;
2056 iod->hosthandle = hosthandle;
2058 schedule_work(&iod->work);
2062 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
2066 * **********************
2067 * Start of FCP handling
2068 * **********************
2072 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2074 struct scatterlist *sg;
2077 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
2082 fod->data_sg_cnt = nent;
2083 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
2084 ((fod->io_dir == NVMET_FCP_WRITE) ?
2085 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2086 /* note: write from initiator perspective */
2087 fod->next_sg = fod->data_sg;
2092 return NVME_SC_INTERNAL;
2096 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2098 if (!fod->data_sg || !fod->data_sg_cnt)
2101 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
2102 ((fod->io_dir == NVMET_FCP_WRITE) ?
2103 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2104 sgl_free(fod->data_sg);
2105 fod->data_sg = NULL;
2106 fod->data_sg_cnt = 0;
2111 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
2115 /* egad, this is ugly. And sqtail is just a best guess */
2116 sqtail = atomic_read(&q->sqtail) % q->sqsize;
2118 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
2119 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
2124 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
2127 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2128 struct nvmet_fc_fcp_iod *fod)
2130 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
2131 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2132 struct nvme_completion *cqe = &ersp->cqe;
2133 u32 *cqewd = (u32 *)cqe;
2134 bool send_ersp = false;
2135 u32 rsn, rspcnt, xfr_length;
2137 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
2138 xfr_length = fod->req.transfer_len;
2140 xfr_length = fod->offset;
2143 * check to see if we can send a 0's rsp.
2144 * Note: to send a 0's response, the NVME-FC host transport will
2145 * recreate the CQE. The host transport knows: sq id, SQHD (last
2146 * seen in an ersp), and command_id. Thus it will create a
2147 * zero-filled CQE with those known fields filled in. Transport
2148 * must send an ersp for any condition where the cqe won't match
2151 * Here are the FC-NVME mandated cases where we must send an ersp:
2152 * every N responses, where N=ersp_ratio
2153 * force fabric commands to send ersp's (not in FC-NVME but good
2155 * normal cmds: any time status is non-zero, or status is zero
2156 * but words 0 or 1 are non-zero.
2157 * the SQ is 90% or more full
2158 * the cmd is a fused command
2159 * transferred data length not equal to cmd iu length
2161 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
2162 if (!(rspcnt % fod->queue->ersp_ratio) ||
2163 nvme_is_fabrics((struct nvme_command *) sqe) ||
2164 xfr_length != fod->req.transfer_len ||
2165 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
2166 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
2167 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
2170 /* re-set the fields */
2171 fod->fcpreq->rspaddr = ersp;
2172 fod->fcpreq->rspdma = fod->rspdma;
2175 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
2176 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
2178 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
2179 rsn = atomic_inc_return(&fod->queue->rsn);
2180 ersp->rsn = cpu_to_be32(rsn);
2181 ersp->xfrd_len = cpu_to_be32(xfr_length);
2182 fod->fcpreq->rsplen = sizeof(*ersp);
2185 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
2186 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
2189 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
2192 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
2193 struct nvmet_fc_fcp_iod *fod)
2195 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2197 /* data no longer needed */
2198 nvmet_fc_free_tgt_pgs(fod);
2201 * if an ABTS was received or we issued the fcp_abort early
2202 * don't call abort routine again.
2204 /* no need to take lock - lock was taken earlier to get here */
2206 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
2208 nvmet_fc_free_fcp_iod(fod->queue, fod);
2212 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2213 struct nvmet_fc_fcp_iod *fod)
2217 fod->fcpreq->op = NVMET_FCOP_RSP;
2218 fod->fcpreq->timeout = 0;
2220 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2222 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2224 nvmet_fc_abort_op(tgtport, fod);
2228 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
2229 struct nvmet_fc_fcp_iod *fod, u8 op)
2231 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2232 struct scatterlist *sg = fod->next_sg;
2233 unsigned long flags;
2234 u32 remaininglen = fod->req.transfer_len - fod->offset;
2239 fcpreq->offset = fod->offset;
2240 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
2243 * for next sequence:
2244 * break at a sg element boundary
2245 * attempt to keep sequence length capped at
2246 * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
2247 * be longer if a single sg element is larger
2248 * than that amount. This is done to avoid creating
2249 * a new sg list to use for the tgtport api.
2253 while (tlen < remaininglen &&
2254 fcpreq->sg_cnt < tgtport->max_sg_cnt &&
2255 tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
2257 tlen += sg_dma_len(sg);
2260 if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
2262 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
2265 if (tlen < remaininglen)
2268 fod->next_sg = NULL;
2270 fcpreq->transfer_length = tlen;
2271 fcpreq->transferred_length = 0;
2272 fcpreq->fcp_error = 0;
2276 * If the last READDATA request: check if LLDD supports
2277 * combined xfr with response.
2279 if ((op == NVMET_FCOP_READDATA) &&
2280 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
2281 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
2282 fcpreq->op = NVMET_FCOP_READDATA_RSP;
2283 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2286 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2289 * should be ok to set w/o lock as its in the thread of
2290 * execution (not an async timer routine) and doesn't
2291 * contend with any clearing action
2295 if (op == NVMET_FCOP_WRITEDATA) {
2296 spin_lock_irqsave(&fod->flock, flags);
2297 fod->writedataactive = false;
2298 spin_unlock_irqrestore(&fod->flock, flags);
2299 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2300 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
2301 fcpreq->fcp_error = ret;
2302 fcpreq->transferred_length = 0;
2303 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
2309 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
2311 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2312 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2314 /* if in the middle of an io and we need to tear down */
2316 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
2317 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2321 nvmet_fc_abort_op(tgtport, fod);
2329 * actual done handler for FCP operations when completed by the lldd
2332 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
2334 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2335 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2336 unsigned long flags;
2339 spin_lock_irqsave(&fod->flock, flags);
2341 fod->writedataactive = false;
2342 spin_unlock_irqrestore(&fod->flock, flags);
2344 switch (fcpreq->op) {
2346 case NVMET_FCOP_WRITEDATA:
2347 if (__nvmet_fc_fod_op_abort(fod, abort))
2349 if (fcpreq->fcp_error ||
2350 fcpreq->transferred_length != fcpreq->transfer_length) {
2351 spin_lock_irqsave(&fod->flock, flags);
2353 spin_unlock_irqrestore(&fod->flock, flags);
2355 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2359 fod->offset += fcpreq->transferred_length;
2360 if (fod->offset != fod->req.transfer_len) {
2361 spin_lock_irqsave(&fod->flock, flags);
2362 fod->writedataactive = true;
2363 spin_unlock_irqrestore(&fod->flock, flags);
2365 /* transfer the next chunk */
2366 nvmet_fc_transfer_fcp_data(tgtport, fod,
2367 NVMET_FCOP_WRITEDATA);
2371 /* data transfer complete, resume with nvmet layer */
2372 fod->req.execute(&fod->req);
2375 case NVMET_FCOP_READDATA:
2376 case NVMET_FCOP_READDATA_RSP:
2377 if (__nvmet_fc_fod_op_abort(fod, abort))
2379 if (fcpreq->fcp_error ||
2380 fcpreq->transferred_length != fcpreq->transfer_length) {
2381 nvmet_fc_abort_op(tgtport, fod);
2387 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2388 /* data no longer needed */
2389 nvmet_fc_free_tgt_pgs(fod);
2390 nvmet_fc_free_fcp_iod(fod->queue, fod);
2394 fod->offset += fcpreq->transferred_length;
2395 if (fod->offset != fod->req.transfer_len) {
2396 /* transfer the next chunk */
2397 nvmet_fc_transfer_fcp_data(tgtport, fod,
2398 NVMET_FCOP_READDATA);
2402 /* data transfer complete, send response */
2404 /* data no longer needed */
2405 nvmet_fc_free_tgt_pgs(fod);
2407 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2411 case NVMET_FCOP_RSP:
2412 if (__nvmet_fc_fod_op_abort(fod, abort))
2414 nvmet_fc_free_fcp_iod(fod->queue, fod);
2423 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2425 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2427 nvmet_fc_fod_op_done(fod);
2431 * actual completion handler after execution by the nvmet layer
2434 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2435 struct nvmet_fc_fcp_iod *fod, int status)
2437 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2438 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2439 unsigned long flags;
2442 spin_lock_irqsave(&fod->flock, flags);
2444 spin_unlock_irqrestore(&fod->flock, flags);
2446 /* if we have a CQE, snoop the last sq_head value */
2448 fod->queue->sqhd = cqe->sq_head;
2451 nvmet_fc_abort_op(tgtport, fod);
2455 /* if an error handling the cmd post initial parsing */
2457 /* fudge up a failed CQE status for our transport error */
2458 memset(cqe, 0, sizeof(*cqe));
2459 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2460 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2461 cqe->command_id = sqe->command_id;
2462 cqe->status = cpu_to_le16(status);
2466 * try to push the data even if the SQE status is non-zero.
2467 * There may be a status where data still was intended to
2470 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2471 /* push the data over before sending rsp */
2472 nvmet_fc_transfer_fcp_data(tgtport, fod,
2473 NVMET_FCOP_READDATA);
2477 /* writes & no data - fall thru */
2480 /* data no longer needed */
2481 nvmet_fc_free_tgt_pgs(fod);
2483 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2488 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2490 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2491 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2493 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2498 * Actual processing routine for received FC-NVME I/O Requests from the LLD
2501 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2502 struct nvmet_fc_fcp_iod *fod)
2504 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2505 u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2509 * Fused commands are currently not supported in the linux
2512 * As such, the implementation of the FC transport does not
2513 * look at the fused commands and order delivery to the upper
2514 * layer until we have both based on csn.
2517 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2519 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2520 fod->io_dir = NVMET_FCP_WRITE;
2521 if (!nvme_is_write(&cmdiu->sqe))
2522 goto transport_error;
2523 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2524 fod->io_dir = NVMET_FCP_READ;
2525 if (nvme_is_write(&cmdiu->sqe))
2526 goto transport_error;
2528 fod->io_dir = NVMET_FCP_NODATA;
2530 goto transport_error;
2533 fod->req.cmd = &fod->cmdiubuf.sqe;
2534 fod->req.cqe = &fod->rspiubuf.cqe;
2536 goto transport_error;
2537 fod->req.port = tgtport->pe->port;
2539 /* clear any response payload */
2540 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2542 fod->data_sg = NULL;
2543 fod->data_sg_cnt = 0;
2545 ret = nvmet_req_init(&fod->req,
2546 &fod->queue->nvme_cq,
2547 &fod->queue->nvme_sq,
2548 &nvmet_fc_tgt_fcp_ops);
2550 /* bad SQE content or invalid ctrl state */
2551 /* nvmet layer has already called op done to send rsp. */
2555 fod->req.transfer_len = xfrlen;
2557 /* keep a running counter of tail position */
2558 atomic_inc(&fod->queue->sqtail);
2560 if (fod->req.transfer_len) {
2561 ret = nvmet_fc_alloc_tgt_pgs(fod);
2563 nvmet_req_complete(&fod->req, ret);
2567 fod->req.sg = fod->data_sg;
2568 fod->req.sg_cnt = fod->data_sg_cnt;
2571 if (fod->io_dir == NVMET_FCP_WRITE) {
2572 /* pull the data over before invoking nvmet layer */
2573 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2580 * can invoke the nvmet_layer now. If read data, cmd completion will
2583 fod->req.execute(&fod->req);
2587 nvmet_fc_abort_op(tgtport, fod);
2591 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2592 * upon the reception of a NVME FCP CMD IU.
2594 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2595 * layer for processing.
2597 * The nvmet_fc layer allocates a local job structure (struct
2598 * nvmet_fc_fcp_iod) from the queue for the io and copies the
2599 * CMD IU buffer to the job structure. As such, on a successful
2600 * completion (returns 0), the LLDD may immediately free/reuse
2601 * the CMD IU buffer passed in the call.
2603 * However, in some circumstances, due to the packetized nature of FC
2604 * and the api of the FC LLDD which may issue a hw command to send the
2605 * response, but the LLDD may not get the hw completion for that command
2606 * and upcall the nvmet_fc layer before a new command may be
2607 * asynchronously received - its possible for a command to be received
2608 * before the LLDD and nvmet_fc have recycled the job structure. It gives
2609 * the appearance of more commands received than fits in the sq.
2610 * To alleviate this scenario, a temporary queue is maintained in the
2611 * transport for pending LLDD requests waiting for a queue job structure.
2612 * In these "overrun" cases, a temporary queue element is allocated
2613 * the LLDD request and CMD iu buffer information remembered, and the
2614 * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2615 * structure is freed, it is immediately reallocated for anything on the
2616 * pending request list. The LLDDs defer_rcv() callback is called,
2617 * informing the LLDD that it may reuse the CMD IU buffer, and the io
2618 * is then started normally with the transport.
2620 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2621 * the completion as successful but must not reuse the CMD IU buffer
2622 * until the LLDD's defer_rcv() callback has been called for the
2623 * corresponding struct nvmefc_tgt_fcp_req pointer.
2625 * If there is any other condition in which an error occurs, the
2626 * transport will return a non-zero status indicating the error.
2627 * In all cases other than -EOVERFLOW, the transport has not accepted the
2628 * request and the LLDD should abort the exchange.
2630 * @target_port: pointer to the (registered) target port the FCP CMD IU
2632 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2633 * the exchange corresponding to the FCP Exchange.
2634 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2635 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2638 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2639 struct nvmefc_tgt_fcp_req *fcpreq,
2640 void *cmdiubuf, u32 cmdiubuf_len)
2642 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2643 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2644 struct nvmet_fc_tgt_queue *queue;
2645 struct nvmet_fc_fcp_iod *fod;
2646 struct nvmet_fc_defer_fcp_req *deferfcp;
2647 unsigned long flags;
2649 /* validate iu, so the connection id can be used to find the queue */
2650 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2651 (cmdiu->format_id != NVME_CMD_FORMAT_ID) ||
2652 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2653 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2656 queue = nvmet_fc_find_target_queue(tgtport,
2657 be64_to_cpu(cmdiu->connection_id));
2662 * note: reference taken by find_target_queue
2663 * After successful fod allocation, the fod will inherit the
2664 * ownership of that reference and will remove the reference
2665 * when the fod is freed.
2668 spin_lock_irqsave(&queue->qlock, flags);
2670 fod = nvmet_fc_alloc_fcp_iod(queue);
2672 spin_unlock_irqrestore(&queue->qlock, flags);
2674 fcpreq->nvmet_fc_private = fod;
2675 fod->fcpreq = fcpreq;
2677 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2679 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2684 if (!tgtport->ops->defer_rcv) {
2685 spin_unlock_irqrestore(&queue->qlock, flags);
2686 /* release the queue lookup reference */
2687 nvmet_fc_tgt_q_put(queue);
2691 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2692 struct nvmet_fc_defer_fcp_req, req_list);
2694 /* Just re-use one that was previously allocated */
2695 list_del(&deferfcp->req_list);
2697 spin_unlock_irqrestore(&queue->qlock, flags);
2699 /* Now we need to dynamically allocate one */
2700 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2702 /* release the queue lookup reference */
2703 nvmet_fc_tgt_q_put(queue);
2706 spin_lock_irqsave(&queue->qlock, flags);
2709 /* For now, use rspaddr / rsplen to save payload information */
2710 fcpreq->rspaddr = cmdiubuf;
2711 fcpreq->rsplen = cmdiubuf_len;
2712 deferfcp->fcp_req = fcpreq;
2714 /* defer processing till a fod becomes available */
2715 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2717 /* NOTE: the queue lookup reference is still valid */
2719 spin_unlock_irqrestore(&queue->qlock, flags);
2723 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2726 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2727 * upon the reception of an ABTS for a FCP command
2729 * Notify the transport that an ABTS has been received for a FCP command
2730 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2731 * LLDD believes the command is still being worked on
2732 * (template_ops->fcp_req_release() has not been called).
2734 * The transport will wait for any outstanding work (an op to the LLDD,
2735 * which the lldd should complete with error due to the ABTS; or the
2736 * completion from the nvmet layer of the nvme command), then will
2737 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2738 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2739 * to the ABTS either after return from this function (assuming any
2740 * outstanding op work has been terminated) or upon the callback being
2743 * @target_port: pointer to the (registered) target port the FCP CMD IU
2745 * @fcpreq: pointer to the fcpreq request structure that corresponds
2746 * to the exchange that received the ABTS.
2749 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2750 struct nvmefc_tgt_fcp_req *fcpreq)
2752 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2753 struct nvmet_fc_tgt_queue *queue;
2754 unsigned long flags;
2756 if (!fod || fod->fcpreq != fcpreq)
2757 /* job appears to have already completed, ignore abort */
2762 spin_lock_irqsave(&queue->qlock, flags);
2765 * mark as abort. The abort handler, invoked upon completion
2766 * of any work, will detect the aborted status and do the
2769 spin_lock(&fod->flock);
2771 fod->aborted = true;
2772 spin_unlock(&fod->flock);
2774 spin_unlock_irqrestore(&queue->qlock, flags);
2776 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2779 struct nvmet_fc_traddr {
2785 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2789 if (match_u64(sstr, &token64))
2797 * This routine validates and extracts the WWN's from the TRADDR string.
2798 * As kernel parsers need the 0x to determine number base, universally
2799 * build string to parse with 0x prefix before parsing name strings.
2802 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2804 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2805 substring_t wwn = { name, &name[sizeof(name)-1] };
2806 int nnoffset, pnoffset;
2808 /* validate if string is one of the 2 allowed formats */
2809 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2810 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2811 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2812 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2813 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2814 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2815 NVME_FC_TRADDR_OXNNLEN;
2816 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2817 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2818 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2819 "pn-", NVME_FC_TRADDR_NNLEN))) {
2820 nnoffset = NVME_FC_TRADDR_NNLEN;
2821 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2827 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2829 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2830 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2833 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2834 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2840 pr_warn("%s: bad traddr string\n", __func__);
2845 nvmet_fc_add_port(struct nvmet_port *port)
2847 struct nvmet_fc_tgtport *tgtport;
2848 struct nvmet_fc_port_entry *pe;
2849 struct nvmet_fc_traddr traddr = { 0L, 0L };
2850 unsigned long flags;
2853 /* validate the address info */
2854 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2855 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2858 /* map the traddr address info to a target port */
2860 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2861 sizeof(port->disc_addr.traddr));
2865 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2870 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2871 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2872 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2873 (tgtport->fc_target_port.port_name == traddr.pn)) {
2874 /* a FC port can only be 1 nvmet port id */
2876 nvmet_fc_portentry_bind(tgtport, pe, port);
2883 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2892 nvmet_fc_remove_port(struct nvmet_port *port)
2894 struct nvmet_fc_port_entry *pe = port->priv;
2896 nvmet_fc_portentry_unbind(pe);
2902 nvmet_fc_discovery_chg(struct nvmet_port *port)
2904 struct nvmet_fc_port_entry *pe = port->priv;
2905 struct nvmet_fc_tgtport *tgtport = pe->tgtport;
2907 if (tgtport && tgtport->ops->discovery_event)
2908 tgtport->ops->discovery_event(&tgtport->fc_target_port);
2911 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2912 .owner = THIS_MODULE,
2913 .type = NVMF_TRTYPE_FC,
2915 .add_port = nvmet_fc_add_port,
2916 .remove_port = nvmet_fc_remove_port,
2917 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2918 .delete_ctrl = nvmet_fc_delete_ctrl,
2919 .discovery_chg = nvmet_fc_discovery_chg,
2922 static int __init nvmet_fc_init_module(void)
2924 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2927 static void __exit nvmet_fc_exit_module(void)
2929 /* sanity check - all lports should be removed */
2930 if (!list_empty(&nvmet_fc_target_list))
2931 pr_warn("%s: targetport list not empty\n", __func__);
2933 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2935 ida_destroy(&nvmet_fc_tgtport_cnt);
2938 module_init(nvmet_fc_init_module);
2939 module_exit(nvmet_fc_exit_module);
2941 MODULE_LICENSE("GPL v2");