target/fc.c

   1 /*
   2  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful.
   9  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
  10  * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
  11  * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
  12  * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
  13  * See the GNU General Public License for more details, a copy of which
  14  * can be found in the file COPYING included with this package
  15  *
  16  */
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18 #include <linux/module.h>
  19 #include <linux/slab.h>
  20 #include <linux/blk-mq.h>
  21 #include <linux/parser.h>
  22 #include <linux/random.h>
  23 #include <uapi/scsi/fc/fc_fs.h>
  24 #include <uapi/scsi/fc/fc_els.h>
  25
  26 #include "nvmet.h"
  27 #include <linux/nvme-fc-driver.h>
  28 #include <linux/nvme-fc.h>
  29
  30
  31 /* *************************** Data Structures/Defines ****************** */
  32
  33
  34 #define NVMET_LS_CTX_COUNT              4
  35
  36 /* for this implementation, assume small single frame rqst/rsp */
  37 #define NVME_FC_MAX_LS_BUFFER_SIZE              2048
  38
  39 struct nvmet_fc_tgtport;
  40 struct nvmet_fc_tgt_assoc;
  41
  42 struct nvmet_fc_ls_iod {
  43         struct nvmefc_tgt_ls_req        *lsreq;
  44         struct nvmefc_tgt_fcp_req       *fcpreq;        /* only if RS */
  45
  46         struct list_head                ls_list;        /* tgtport->ls_list */
  47
  48         struct nvmet_fc_tgtport         *tgtport;
  49         struct nvmet_fc_tgt_assoc       *assoc;
  50
  51         u8                              *rqstbuf;
  52         u8                              *rspbuf;
  53         u16                             rqstdatalen;
  54         dma_addr_t                      rspdma;
  55
  56         struct scatterlist              sg[2];
  57
  58         struct work_struct              work;
  59 } __aligned(sizeof(unsigned long long));
  60
  61 /* desired maximum for a single sequence - if sg list allows it */
  62 #define NVMET_FC_MAX_SEQ_LENGTH         (256 * 1024)
  63
  64 enum nvmet_fcp_datadir {
  65         NVMET_FCP_NODATA,
  66         NVMET_FCP_WRITE,
  67         NVMET_FCP_READ,
  68         NVMET_FCP_ABORTED,
  69 };
  70
  71 struct nvmet_fc_fcp_iod {
  72         struct nvmefc_tgt_fcp_req       *fcpreq;
  73
  74         struct nvme_fc_cmd_iu           cmdiubuf;
  75         struct nvme_fc_ersp_iu          rspiubuf;
  76         dma_addr_t                      rspdma;
  77         struct scatterlist              *next_sg;
  78         struct scatterlist              *data_sg;
  79         int                             data_sg_cnt;
  80         u32                             total_length;
  81         u32                             offset;
  82         enum nvmet_fcp_datadir          io_dir;
  83         bool                            active;
  84         bool                            abort;
  85         bool                            aborted;
  86         bool                            writedataactive;
  87         spinlock_t                      flock;
  88
  89         struct nvmet_req                req;
  90         struct work_struct              work;
  91         struct work_struct              done_work;
  92
  93         struct nvmet_fc_tgtport         *tgtport;
  94         struct nvmet_fc_tgt_queue       *queue;
  95
  96         struct list_head                fcp_list;       /* tgtport->fcp_list */
  97 };
  98
  99 struct nvmet_fc_tgtport {
 100
 101         struct nvmet_fc_target_port     fc_target_port;
 102
 103         struct list_head                tgt_list; /* nvmet_fc_target_list */
 104         struct device                   *dev;   /* dev for dma mapping */
 105         struct nvmet_fc_target_template *ops;
 106
 107         struct nvmet_fc_ls_iod          *iod;
 108         spinlock_t                      lock;
 109         struct list_head                ls_list;
 110         struct list_head                ls_busylist;
 111         struct list_head                assoc_list;
 112         struct ida                      assoc_cnt;
 113         struct nvmet_port               *port;
 114         struct kref                     ref;
 115         u32                             max_sg_cnt;
 116 };
 117
 118 struct nvmet_fc_defer_fcp_req {
 119         struct list_head                req_list;
 120         struct nvmefc_tgt_fcp_req       *fcp_req;
 121 };
 122
 123 struct nvmet_fc_tgt_queue {
 124         bool                            ninetypercent;
 125         u16                             qid;
 126         u16                             sqsize;
 127         u16                             ersp_ratio;
 128         __le16                          sqhd;
 129         int                             cpu;
 130         atomic_t                        connected;
 131         atomic_t                        sqtail;
 132         atomic_t                        zrspcnt;
 133         atomic_t                        rsn;
 134         spinlock_t                      qlock;
 135         struct nvmet_port               *port;
 136         struct nvmet_cq                 nvme_cq;
 137         struct nvmet_sq                 nvme_sq;
 138         struct nvmet_fc_tgt_assoc       *assoc;
 139         struct nvmet_fc_fcp_iod         *fod;           /* array of fcp_iods */
 140         struct list_head                fod_list;
 141         struct list_head                pending_cmd_list;
 142         struct list_head                avail_defer_list;
 143         struct workqueue_struct         *work_q;
 144         struct kref                     ref;
 145 } __aligned(sizeof(unsigned long long));
 146
 147 struct nvmet_fc_tgt_assoc {
 148         u64                             association_id;
 149         u32                             a_id;
 150         struct nvmet_fc_tgtport         *tgtport;
 151         struct list_head                a_list;
 152         struct nvmet_fc_tgt_queue       *queues[NVMET_NR_QUEUES + 1];
 153         struct kref                     ref;
 154 };
 155
 156
 157 static inline int
 158 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
 159 {
 160         return (iodptr - iodptr->tgtport->iod);
 161 }
 162
 163 static inline int
 164 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
 165 {
 166         return (fodptr - fodptr->queue->fod);
 167 }
 168
 169
 170 /*
 171  * Association and Connection IDs:
 172  *
 173  * Association ID will have random number in upper 6 bytes and zero
 174  *   in lower 2 bytes
 175  *
 176  * Connection IDs will be Association ID with QID or'd in lower 2 bytes
 177  *
 178  * note: Association ID = Connection ID for queue 0
 179  */
 180 #define BYTES_FOR_QID                   sizeof(u16)
 181 #define BYTES_FOR_QID_SHIFT             (BYTES_FOR_QID * 8)
 182 #define NVMET_FC_QUEUEID_MASK           ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
 183
 184 static inline u64
 185 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
 186 {
 187         return (assoc->association_id | qid);
 188 }
 189
 190 static inline u64
 191 nvmet_fc_getassociationid(u64 connectionid)
 192 {
 193         return connectionid & ~NVMET_FC_QUEUEID_MASK;
 194 }
 195
 196 static inline u16
 197 nvmet_fc_getqueueid(u64 connectionid)
 198 {
 199         return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
 200 }
 201
 202 static inline struct nvmet_fc_tgtport *
 203 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
 204 {
 205         return container_of(targetport, struct nvmet_fc_tgtport,
 206                                  fc_target_port);
 207 }
 208
 209 static inline struct nvmet_fc_fcp_iod *
 210 nvmet_req_to_fod(struct nvmet_req *nvme_req)
 211 {
 212         return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
 213 }
 214
 215
 216 /* *************************** Globals **************************** */
 217
 218
 219 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
 220
 221 static LIST_HEAD(nvmet_fc_target_list);
 222 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
 223
 224
 225 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
 226 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
 227 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work);
 228 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
 229 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
 230 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
 231 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
 232 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
 233 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
 234 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
 235                                         struct nvmet_fc_fcp_iod *fod);
 236
 237
 238 /* *********************** FC-NVME DMA Handling **************************** */
 239
 240 /*
 241  * The fcloop device passes in a NULL device pointer. Real LLD's will
 242  * pass in a valid device pointer. If NULL is passed to the dma mapping
 243  * routines, depending on the platform, it may or may not succeed, and
 244  * may crash.
 245  *
 246  * As such:
 247  * Wrapper all the dma routines and check the dev pointer.
 248  *
 249  * If simple mappings (return just a dma address, we'll noop them,
 250  * returning a dma address of 0.
 251  *
 252  * On more complex mappings (dma_map_sg), a pseudo routine fills
 253  * in the scatter list, setting all dma addresses to 0.
 254  */
 255
 256 static inline dma_addr_t
 257 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 258                 enum dma_data_direction dir)
 259 {
 260         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 261 }
 262
 263 static inline int
 264 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 265 {
 266         return dev ? dma_mapping_error(dev, dma_addr) : 0;
 267 }
 268
 269 static inline void
 270 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 271         enum dma_data_direction dir)
 272 {
 273         if (dev)
 274                 dma_unmap_single(dev, addr, size, dir);
 275 }
 276
 277 static inline void
 278 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 279                 enum dma_data_direction dir)
 280 {
 281         if (dev)
 282                 dma_sync_single_for_cpu(dev, addr, size, dir);
 283 }
 284
 285 static inline void
 286 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 287                 enum dma_data_direction dir)
 288 {
 289         if (dev)
 290                 dma_sync_single_for_device(dev, addr, size, dir);
 291 }
 292
 293 /* pseudo dma_map_sg call */
 294 static int
 295 fc_map_sg(struct scatterlist *sg, int nents)
 296 {
 297         struct scatterlist *s;
 298         int i;
 299
 300         WARN_ON(nents == 0 || sg[0].length == 0);
 301
 302         for_each_sg(sg, s, nents, i) {
 303                 s->dma_address = 0L;
 304 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 305                 s->dma_length = s->length;
 306 #endif
 307         }
 308         return nents;
 309 }
 310
 311 static inline int
 312 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 313                 enum dma_data_direction dir)
 314 {
 315         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
 316 }
 317
 318 static inline void
 319 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 320                 enum dma_data_direction dir)
 321 {
 322         if (dev)
 323                 dma_unmap_sg(dev, sg, nents, dir);
 324 }
 325
 326
 327 /* *********************** FC-NVME Port Management ************************ */
 328
 329
 330 static int
 331 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
 332 {
 333         struct nvmet_fc_ls_iod *iod;
 334         int i;
 335
 336         iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
 337                         GFP_KERNEL);
 338         if (!iod)
 339                 return -ENOMEM;
 340
 341         tgtport->iod = iod;
 342
 343         for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
 344                 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
 345                 iod->tgtport = tgtport;
 346                 list_add_tail(&iod->ls_list, &tgtport->ls_list);
 347
 348                 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
 349                         GFP_KERNEL);
 350                 if (!iod->rqstbuf)
 351                         goto out_fail;
 352
 353                 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
 354
 355                 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
 356                                                 NVME_FC_MAX_LS_BUFFER_SIZE,
 357                                                 DMA_TO_DEVICE);
 358                 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
 359                         goto out_fail;
 360         }
 361
 362         return 0;
 363
 364 out_fail:
 365         kfree(iod->rqstbuf);
 366         list_del(&iod->ls_list);
 367         for (iod--, i--; i >= 0; iod--, i--) {
 368                 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
 369                                 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
 370                 kfree(iod->rqstbuf);
 371                 list_del(&iod->ls_list);
 372         }
 373
 374         kfree(iod);
 375
 376         return -EFAULT;
 377 }
 378
 379 static void
 380 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
 381 {
 382         struct nvmet_fc_ls_iod *iod = tgtport->iod;
 383         int i;
 384
 385         for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
 386                 fc_dma_unmap_single(tgtport->dev,
 387                                 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
 388                                 DMA_TO_DEVICE);
 389                 kfree(iod->rqstbuf);
 390                 list_del(&iod->ls_list);
 391         }
 392         kfree(tgtport->iod);
 393 }
 394
 395 static struct nvmet_fc_ls_iod *
 396 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
 397 {
 398         struct nvmet_fc_ls_iod *iod;
 399         unsigned long flags;
 400
 401         spin_lock_irqsave(&tgtport->lock, flags);
 402         iod = list_first_entry_or_null(&tgtport->ls_list,
 403                                         struct nvmet_fc_ls_iod, ls_list);
 404         if (iod)
 405                 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
 406         spin_unlock_irqrestore(&tgtport->lock, flags);
 407         return iod;
 408 }
 409
 410
 411 static void
 412 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
 413                         struct nvmet_fc_ls_iod *iod)
 414 {
 415         unsigned long flags;
 416
 417         spin_lock_irqsave(&tgtport->lock, flags);
 418         list_move(&iod->ls_list, &tgtport->ls_list);
 419         spin_unlock_irqrestore(&tgtport->lock, flags);
 420 }
 421
 422 static void
 423 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
 424                                 struct nvmet_fc_tgt_queue *queue)
 425 {
 426         struct nvmet_fc_fcp_iod *fod = queue->fod;
 427         int i;
 428
 429         for (i = 0; i < queue->sqsize; fod++, i++) {
 430                 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
 431                 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work);
 432                 fod->tgtport = tgtport;
 433                 fod->queue = queue;
 434                 fod->active = false;
 435                 fod->abort = false;
 436                 fod->aborted = false;
 437                 fod->fcpreq = NULL;
 438                 list_add_tail(&fod->fcp_list, &queue->fod_list);
 439                 spin_lock_init(&fod->flock);
 440
 441                 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
 442                                         sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 443                 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
 444                         list_del(&fod->fcp_list);
 445                         for (fod--, i--; i >= 0; fod--, i--) {
 446                                 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
 447                                                 sizeof(fod->rspiubuf),
 448                                                 DMA_TO_DEVICE);
 449                                 fod->rspdma = 0L;
 450                                 list_del(&fod->fcp_list);
 451                         }
 452
 453                         return;
 454                 }
 455         }
 456 }
 457
 458 static void
 459 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
 460                                 struct nvmet_fc_tgt_queue *queue)
 461 {
 462         struct nvmet_fc_fcp_iod *fod = queue->fod;
 463         int i;
 464
 465         for (i = 0; i < queue->sqsize; fod++, i++) {
 466                 if (fod->rspdma)
 467                         fc_dma_unmap_single(tgtport->dev, fod->rspdma,
 468                                 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 469         }
 470 }
 471
 472 static struct nvmet_fc_fcp_iod *
 473 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
 474 {
 475         struct nvmet_fc_fcp_iod *fod;
 476
 477         lockdep_assert_held(&queue->qlock);
 478
 479         fod = list_first_entry_or_null(&queue->fod_list,
 480                                         struct nvmet_fc_fcp_iod, fcp_list);
 481         if (fod) {
 482                 list_del(&fod->fcp_list);
 483                 fod->active = true;
 484                 /*
 485                  * no queue reference is taken, as it was taken by the
 486                  * queue lookup just prior to the allocation. The iod
 487                  * will "inherit" that reference.
 488                  */
 489         }
 490         return fod;
 491 }
 492
 493
 494 static void
 495 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
 496                        struct nvmet_fc_tgt_queue *queue,
 497                        struct nvmefc_tgt_fcp_req *fcpreq)
 498 {
 499         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
 500
 501         /*
 502          * put all admin cmds on hw queue id 0. All io commands go to
 503          * the respective hw queue based on a modulo basis
 504          */
 505         fcpreq->hwqid = queue->qid ?
 506                         ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
 507
 508         if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR)
 509                 queue_work_on(queue->cpu, queue->work_q, &fod->work);
 510         else
 511                 nvmet_fc_handle_fcp_rqst(tgtport, fod);
 512 }
 513
 514 static void
 515 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
 516                         struct nvmet_fc_fcp_iod *fod)
 517 {
 518         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
 519         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
 520         struct nvmet_fc_defer_fcp_req *deferfcp;
 521         unsigned long flags;
 522
 523         fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
 524                                 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 525
 526         fcpreq->nvmet_fc_private = NULL;
 527
 528         fod->active = false;
 529         fod->abort = false;
 530         fod->aborted = false;
 531         fod->writedataactive = false;
 532         fod->fcpreq = NULL;
 533
 534         tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
 535
 536         /* release the queue lookup reference on the completed IO */
 537         nvmet_fc_tgt_q_put(queue);
 538
 539         spin_lock_irqsave(&queue->qlock, flags);
 540         deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
 541                                 struct nvmet_fc_defer_fcp_req, req_list);
 542         if (!deferfcp) {
 543                 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
 544                 spin_unlock_irqrestore(&queue->qlock, flags);
 545                 return;
 546         }
 547
 548         /* Re-use the fod for the next pending cmd that was deferred */
 549         list_del(&deferfcp->req_list);
 550
 551         fcpreq = deferfcp->fcp_req;
 552
 553         /* deferfcp can be reused for another IO at a later date */
 554         list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
 555
 556         spin_unlock_irqrestore(&queue->qlock, flags);
 557
 558         /* Save NVME CMD IO in fod */
 559         memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
 560
 561         /* Setup new fcpreq to be processed */
 562         fcpreq->rspaddr = NULL;
 563         fcpreq->rsplen  = 0;
 564         fcpreq->nvmet_fc_private = fod;
 565         fod->fcpreq = fcpreq;
 566         fod->active = true;
 567
 568         /* inform LLDD IO is now being processed */
 569         tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
 570
 571         /* Submit deferred IO for processing */
 572         nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
 573
 574         /*
 575          * Leave the queue lookup get reference taken when
 576          * fod was originally allocated.
 577          */
 578 }
 579
 580 static int
 581 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
 582 {
 583         int cpu, idx, cnt;
 584
 585         if (tgtport->ops->max_hw_queues == 1)
 586                 return WORK_CPU_UNBOUND;
 587
 588         /* Simple cpu selection based on qid modulo active cpu count */
 589         idx = !qid ? 0 : (qid - 1) % num_active_cpus();
 590
 591         /* find the n'th active cpu */
 592         for (cpu = 0, cnt = 0; ; ) {
 593                 if (cpu_active(cpu)) {
 594                         if (cnt == idx)
 595                                 break;
 596                         cnt++;
 597                 }
 598                 cpu = (cpu + 1) % num_possible_cpus();
 599         }
 600
 601         return cpu;
 602 }
 603
 604 static struct nvmet_fc_tgt_queue *
 605 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
 606                         u16 qid, u16 sqsize)
 607 {
 608         struct nvmet_fc_tgt_queue *queue;
 609         unsigned long flags;
 610         int ret;
 611
 612         if (qid > NVMET_NR_QUEUES)
 613                 return NULL;
 614
 615         queue = kzalloc((sizeof(*queue) +
 616                                 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
 617                                 GFP_KERNEL);
 618         if (!queue)
 619                 return NULL;
 620
 621         if (!nvmet_fc_tgt_a_get(assoc))
 622                 goto out_free_queue;
 623
 624         queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
 625                                 assoc->tgtport->fc_target_port.port_num,
 626                                 assoc->a_id, qid);
 627         if (!queue->work_q)
 628                 goto out_a_put;
 629
 630         queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
 631         queue->qid = qid;
 632         queue->sqsize = sqsize;
 633         queue->assoc = assoc;
 634         queue->port = assoc->tgtport->port;
 635         queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
 636         INIT_LIST_HEAD(&queue->fod_list);
 637         INIT_LIST_HEAD(&queue->avail_defer_list);
 638         INIT_LIST_HEAD(&queue->pending_cmd_list);
 639         atomic_set(&queue->connected, 0);
 640         atomic_set(&queue->sqtail, 0);
 641         atomic_set(&queue->rsn, 1);
 642         atomic_set(&queue->zrspcnt, 0);
 643         spin_lock_init(&queue->qlock);
 644         kref_init(&queue->ref);
 645
 646         nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
 647
 648         ret = nvmet_sq_init(&queue->nvme_sq);
 649         if (ret)
 650                 goto out_fail_iodlist;
 651
 652         WARN_ON(assoc->queues[qid]);
 653         spin_lock_irqsave(&assoc->tgtport->lock, flags);
 654         assoc->queues[qid] = queue;
 655         spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
 656
 657         return queue;
 658
 659 out_fail_iodlist:
 660         nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
 661         destroy_workqueue(queue->work_q);
 662 out_a_put:
 663         nvmet_fc_tgt_a_put(assoc);
 664 out_free_queue:
 665         kfree(queue);
 666         return NULL;
 667 }
 668
 669
 670 static void
 671 nvmet_fc_tgt_queue_free(struct kref *ref)
 672 {
 673         struct nvmet_fc_tgt_queue *queue =
 674                 container_of(ref, struct nvmet_fc_tgt_queue, ref);
 675         unsigned long flags;
 676
 677         spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
 678         queue->assoc->queues[queue->qid] = NULL;
 679         spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
 680
 681         nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
 682
 683         nvmet_fc_tgt_a_put(queue->assoc);
 684
 685         destroy_workqueue(queue->work_q);
 686
 687         kfree(queue);
 688 }
 689
 690 static void
 691 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
 692 {
 693         kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
 694 }
 695
 696 static int
 697 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
 698 {
 699         return kref_get_unless_zero(&queue->ref);
 700 }
 701
 702
 703 static void
 704 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
 705 {
 706         struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
 707         struct nvmet_fc_fcp_iod *fod = queue->fod;
 708         struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
 709         unsigned long flags;
 710         int i, writedataactive;
 711         bool disconnect;
 712
 713         disconnect = atomic_xchg(&queue->connected, 0);
 714
 715         spin_lock_irqsave(&queue->qlock, flags);
 716         /* about outstanding io's */
 717         for (i = 0; i < queue->sqsize; fod++, i++) {
 718                 if (fod->active) {
 719                         spin_lock(&fod->flock);
 720                         fod->abort = true;
 721                         writedataactive = fod->writedataactive;
 722                         spin_unlock(&fod->flock);
 723                         /*
 724                          * only call lldd abort routine if waiting for
 725                          * writedata. other outstanding ops should finish
 726                          * on their own.
 727                          */
 728                         if (writedataactive) {
 729                                 spin_lock(&fod->flock);
 730                                 fod->aborted = true;
 731                                 spin_unlock(&fod->flock);
 732                                 tgtport->ops->fcp_abort(
 733                                         &tgtport->fc_target_port, fod->fcpreq);
 734                         }
 735                 }
 736         }
 737
 738         /* Cleanup defer'ed IOs in queue */
 739         list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
 740                                 req_list) {
 741                 list_del(&deferfcp->req_list);
 742                 kfree(deferfcp);
 743         }
 744
 745         for (;;) {
 746                 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
 747                                 struct nvmet_fc_defer_fcp_req, req_list);
 748                 if (!deferfcp)
 749                         break;
 750
 751                 list_del(&deferfcp->req_list);
 752                 spin_unlock_irqrestore(&queue->qlock, flags);
 753
 754                 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
 755                                 deferfcp->fcp_req);
 756
 757                 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
 758                                 deferfcp->fcp_req);
 759
 760                 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
 761                                 deferfcp->fcp_req);
 762
 763                 /* release the queue lookup reference */
 764                 nvmet_fc_tgt_q_put(queue);
 765
 766                 kfree(deferfcp);
 767
 768                 spin_lock_irqsave(&queue->qlock, flags);
 769         }
 770         spin_unlock_irqrestore(&queue->qlock, flags);
 771
 772         flush_workqueue(queue->work_q);
 773
 774         if (disconnect)
 775                 nvmet_sq_destroy(&queue->nvme_sq);
 776
 777         nvmet_fc_tgt_q_put(queue);
 778 }
 779
 780 static struct nvmet_fc_tgt_queue *
 781 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
 782                                 u64 connection_id)
 783 {
 784         struct nvmet_fc_tgt_assoc *assoc;
 785         struct nvmet_fc_tgt_queue *queue;
 786         u64 association_id = nvmet_fc_getassociationid(connection_id);
 787         u16 qid = nvmet_fc_getqueueid(connection_id);
 788         unsigned long flags;
 789
 790         if (qid > NVMET_NR_QUEUES)
 791                 return NULL;
 792
 793         spin_lock_irqsave(&tgtport->lock, flags);
 794         list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
 795                 if (association_id == assoc->association_id) {
 796                         queue = assoc->queues[qid];
 797                         if (queue &&
 798                             (!atomic_read(&queue->connected) ||
 799                              !nvmet_fc_tgt_q_get(queue)))
 800                                 queue = NULL;
 801                         spin_unlock_irqrestore(&tgtport->lock, flags);
 802                         return queue;
 803                 }
 804         }
 805         spin_unlock_irqrestore(&tgtport->lock, flags);
 806         return NULL;
 807 }
 808
 809 static struct nvmet_fc_tgt_assoc *
 810 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
 811 {
 812         struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
 813         unsigned long flags;
 814         u64 ran;
 815         int idx;
 816         bool needrandom = true;
 817
 818         assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
 819         if (!assoc)
 820                 return NULL;
 821
 822         idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
 823         if (idx < 0)
 824                 goto out_free_assoc;
 825
 826         if (!nvmet_fc_tgtport_get(tgtport))
 827                 goto out_ida_put;
 828
 829         assoc->tgtport = tgtport;
 830         assoc->a_id = idx;
 831         INIT_LIST_HEAD(&assoc->a_list);
 832         kref_init(&assoc->ref);
 833
 834         while (needrandom) {
 835                 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
 836                 ran = ran << BYTES_FOR_QID_SHIFT;
 837
 838                 spin_lock_irqsave(&tgtport->lock, flags);
 839                 needrandom = false;
 840                 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
 841                         if (ran == tmpassoc->association_id) {
 842                                 needrandom = true;
 843                                 break;
 844                         }
 845                 if (!needrandom) {
 846                         assoc->association_id = ran;
 847                         list_add_tail(&assoc->a_list, &tgtport->assoc_list);
 848                 }
 849                 spin_unlock_irqrestore(&tgtport->lock, flags);
 850         }
 851
 852         return assoc;
 853
 854 out_ida_put:
 855         ida_simple_remove(&tgtport->assoc_cnt, idx);
 856 out_free_assoc:
 857         kfree(assoc);
 858         return NULL;
 859 }
 860
 861 static void
 862 nvmet_fc_target_assoc_free(struct kref *ref)
 863 {
 864         struct nvmet_fc_tgt_assoc *assoc =
 865                 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
 866         struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
 867         unsigned long flags;
 868
 869         spin_lock_irqsave(&tgtport->lock, flags);
 870         list_del(&assoc->a_list);
 871         spin_unlock_irqrestore(&tgtport->lock, flags);
 872         ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
 873         kfree(assoc);
 874         nvmet_fc_tgtport_put(tgtport);
 875 }
 876
 877 static void
 878 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
 879 {
 880         kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
 881 }
 882
 883 static int
 884 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
 885 {
 886         return kref_get_unless_zero(&assoc->ref);
 887 }
 888
 889 static void
 890 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
 891 {
 892         struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
 893         struct nvmet_fc_tgt_queue *queue;
 894         unsigned long flags;
 895         int i;
 896
 897         spin_lock_irqsave(&tgtport->lock, flags);
 898         for (i = NVMET_NR_QUEUES; i >= 0; i--) {
 899                 queue = assoc->queues[i];
 900                 if (queue) {
 901                         if (!nvmet_fc_tgt_q_get(queue))
 902                                 continue;
 903                         spin_unlock_irqrestore(&tgtport->lock, flags);
 904                         nvmet_fc_delete_target_queue(queue);
 905                         nvmet_fc_tgt_q_put(queue);
 906                         spin_lock_irqsave(&tgtport->lock, flags);
 907                 }
 908         }
 909         spin_unlock_irqrestore(&tgtport->lock, flags);
 910
 911         nvmet_fc_tgt_a_put(assoc);
 912 }
 913
 914 static struct nvmet_fc_tgt_assoc *
 915 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
 916                                 u64 association_id)
 917 {
 918         struct nvmet_fc_tgt_assoc *assoc;
 919         struct nvmet_fc_tgt_assoc *ret = NULL;
 920         unsigned long flags;
 921
 922         spin_lock_irqsave(&tgtport->lock, flags);
 923         list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
 924                 if (association_id == assoc->association_id) {
 925                         ret = assoc;
 926                         nvmet_fc_tgt_a_get(assoc);
 927                         break;
 928                 }
 929         }
 930         spin_unlock_irqrestore(&tgtport->lock, flags);
 931
 932         return ret;
 933 }
 934
 935
 936 /**
 937  * nvme_fc_register_targetport - transport entry point called by an
 938  *                              LLDD to register the existence of a local
 939  *                              NVME subystem FC port.
 940  * @pinfo:     pointer to information about the port to be registered
 941  * @template:  LLDD entrypoints and operational parameters for the port
 942  * @dev:       physical hardware device node port corresponds to. Will be
 943  *             used for DMA mappings
 944  * @portptr:   pointer to a local port pointer. Upon success, the routine
 945  *             will allocate a nvme_fc_local_port structure and place its
 946  *             address in the local port pointer. Upon failure, local port
 947  *             pointer will be set to NULL.
 948  *
 949  * Returns:
 950  * a completion status. Must be 0 upon success; a negative errno
 951  * (ex: -ENXIO) upon failure.
 952  */
 953 int
 954 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
 955                         struct nvmet_fc_target_template *template,
 956                         struct device *dev,
 957                         struct nvmet_fc_target_port **portptr)
 958 {
 959         struct nvmet_fc_tgtport *newrec;
 960         unsigned long flags;
 961         int ret, idx;
 962
 963         if (!template->xmt_ls_rsp || !template->fcp_op ||
 964             !template->fcp_abort ||
 965             !template->fcp_req_release || !template->targetport_delete ||
 966             !template->max_hw_queues || !template->max_sgl_segments ||
 967             !template->max_dif_sgl_segments || !template->dma_boundary) {
 968                 ret = -EINVAL;
 969                 goto out_regtgt_failed;
 970         }
 971
 972         newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
 973                          GFP_KERNEL);
 974         if (!newrec) {
 975                 ret = -ENOMEM;
 976                 goto out_regtgt_failed;
 977         }
 978
 979         idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
 980         if (idx < 0) {
 981                 ret = -ENOSPC;
 982                 goto out_fail_kfree;
 983         }
 984
 985         if (!get_device(dev) && dev) {
 986                 ret = -ENODEV;
 987                 goto out_ida_put;
 988         }
 989
 990         newrec->fc_target_port.node_name = pinfo->node_name;
 991         newrec->fc_target_port.port_name = pinfo->port_name;
 992         newrec->fc_target_port.private = &newrec[1];
 993         newrec->fc_target_port.port_id = pinfo->port_id;
 994         newrec->fc_target_port.port_num = idx;
 995         INIT_LIST_HEAD(&newrec->tgt_list);
 996         newrec->dev = dev;
 997         newrec->ops = template;
 998         spin_lock_init(&newrec->lock);
 999         INIT_LIST_HEAD(&newrec->ls_list);
1000         INIT_LIST_HEAD(&newrec->ls_busylist);
1001         INIT_LIST_HEAD(&newrec->assoc_list);
1002         kref_init(&newrec->ref);
1003         ida_init(&newrec->assoc_cnt);
1004         newrec->max_sg_cnt = template->max_sgl_segments;
1005
1006         ret = nvmet_fc_alloc_ls_iodlist(newrec);
1007         if (ret) {
1008                 ret = -ENOMEM;
1009                 goto out_free_newrec;
1010         }
1011
1012         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1013         list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1014         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1015
1016         *portptr = &newrec->fc_target_port;
1017         return 0;
1018
1019 out_free_newrec:
1020         put_device(dev);
1021 out_ida_put:
1022         ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1023 out_fail_kfree:
1024         kfree(newrec);
1025 out_regtgt_failed:
1026         *portptr = NULL;
1027         return ret;
1028 }
1029 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1030
1031
1032 static void
1033 nvmet_fc_free_tgtport(struct kref *ref)
1034 {
1035         struct nvmet_fc_tgtport *tgtport =
1036                 container_of(ref, struct nvmet_fc_tgtport, ref);
1037         struct device *dev = tgtport->dev;
1038         unsigned long flags;
1039
1040         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1041         list_del(&tgtport->tgt_list);
1042         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1043
1044         nvmet_fc_free_ls_iodlist(tgtport);
1045
1046         /* let the LLDD know we've finished tearing it down */
1047         tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1048
1049         ida_simple_remove(&nvmet_fc_tgtport_cnt,
1050                         tgtport->fc_target_port.port_num);
1051
1052         ida_destroy(&tgtport->assoc_cnt);
1053
1054         kfree(tgtport);
1055
1056         put_device(dev);
1057 }
1058
1059 static void
1060 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1061 {
1062         kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1063 }
1064
1065 static int
1066 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1067 {
1068         return kref_get_unless_zero(&tgtport->ref);
1069 }
1070
1071 static void
1072 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1073 {
1074         struct nvmet_fc_tgt_assoc *assoc, *next;
1075         unsigned long flags;
1076
1077         spin_lock_irqsave(&tgtport->lock, flags);
1078         list_for_each_entry_safe(assoc, next,
1079                                 &tgtport->assoc_list, a_list) {
1080                 if (!nvmet_fc_tgt_a_get(assoc))
1081                         continue;
1082                 spin_unlock_irqrestore(&tgtport->lock, flags);
1083                 nvmet_fc_delete_target_assoc(assoc);
1084                 nvmet_fc_tgt_a_put(assoc);
1085                 spin_lock_irqsave(&tgtport->lock, flags);
1086         }
1087         spin_unlock_irqrestore(&tgtport->lock, flags);
1088 }
1089
1090 /*
1091  * nvmet layer has called to terminate an association
1092  */
1093 static void
1094 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1095 {
1096         struct nvmet_fc_tgtport *tgtport, *next;
1097         struct nvmet_fc_tgt_assoc *assoc;
1098         struct nvmet_fc_tgt_queue *queue;
1099         unsigned long flags;
1100         bool found_ctrl = false;
1101
1102         /* this is a bit ugly, but don't want to make locks layered */
1103         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1104         list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1105                         tgt_list) {
1106                 if (!nvmet_fc_tgtport_get(tgtport))
1107                         continue;
1108                 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1109
1110                 spin_lock_irqsave(&tgtport->lock, flags);
1111                 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1112                         queue = assoc->queues[0];
1113                         if (queue && queue->nvme_sq.ctrl == ctrl) {
1114                                 if (nvmet_fc_tgt_a_get(assoc))
1115                                         found_ctrl = true;
1116                                 break;
1117                         }
1118                 }
1119                 spin_unlock_irqrestore(&tgtport->lock, flags);
1120
1121                 nvmet_fc_tgtport_put(tgtport);
1122
1123                 if (found_ctrl) {
1124                         nvmet_fc_delete_target_assoc(assoc);
1125                         nvmet_fc_tgt_a_put(assoc);
1126                         return;
1127                 }
1128
1129                 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1130         }
1131         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1132 }
1133
1134 /**
1135  * nvme_fc_unregister_targetport - transport entry point called by an
1136  *                              LLDD to deregister/remove a previously
1137  *                              registered a local NVME subsystem FC port.
1138  * @tgtport: pointer to the (registered) target port that is to be
1139  *           deregistered.
1140  *
1141  * Returns:
1142  * a completion status. Must be 0 upon success; a negative errno
1143  * (ex: -ENXIO) upon failure.
1144  */
1145 int
1146 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1147 {
1148         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1149
1150         /* terminate any outstanding associations */
1151         __nvmet_fc_free_assocs(tgtport);
1152
1153         nvmet_fc_tgtport_put(tgtport);
1154
1155         return 0;
1156 }
1157 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1158
1159
1160 /* *********************** FC-NVME LS Handling **************************** */
1161
1162
1163 static void
1164 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1165 {
1166         struct fcnvme_ls_acc_hdr *acc = buf;
1167
1168         acc->w0.ls_cmd = ls_cmd;
1169         acc->desc_list_len = desc_len;
1170         acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1171         acc->rqst.desc_len =
1172                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1173         acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1174 }
1175
1176 static int
1177 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1178                         u8 reason, u8 explanation, u8 vendor)
1179 {
1180         struct fcnvme_ls_rjt *rjt = buf;
1181
1182         nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1183                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1184                         ls_cmd);
1185         rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1186         rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1187         rjt->rjt.reason_code = reason;
1188         rjt->rjt.reason_explanation = explanation;
1189         rjt->rjt.vendor = vendor;
1190
1191         return sizeof(struct fcnvme_ls_rjt);
1192 }
1193
1194 /* Validation Error indexes into the string table below */
1195 enum {
1196         VERR_NO_ERROR           = 0,
1197         VERR_CR_ASSOC_LEN       = 1,
1198         VERR_CR_ASSOC_RQST_LEN  = 2,
1199         VERR_CR_ASSOC_CMD       = 3,
1200         VERR_CR_ASSOC_CMD_LEN   = 4,
1201         VERR_ERSP_RATIO         = 5,
1202         VERR_ASSOC_ALLOC_FAIL   = 6,
1203         VERR_QUEUE_ALLOC_FAIL   = 7,
1204         VERR_CR_CONN_LEN        = 8,
1205         VERR_CR_CONN_RQST_LEN   = 9,
1206         VERR_ASSOC_ID           = 10,
1207         VERR_ASSOC_ID_LEN       = 11,
1208         VERR_NO_ASSOC           = 12,
1209         VERR_CONN_ID            = 13,
1210         VERR_CONN_ID_LEN        = 14,
1211         VERR_NO_CONN            = 15,
1212         VERR_CR_CONN_CMD        = 16,
1213         VERR_CR_CONN_CMD_LEN    = 17,
1214         VERR_DISCONN_LEN        = 18,
1215         VERR_DISCONN_RQST_LEN   = 19,
1216         VERR_DISCONN_CMD        = 20,
1217         VERR_DISCONN_CMD_LEN    = 21,
1218         VERR_DISCONN_SCOPE      = 22,
1219         VERR_RS_LEN             = 23,
1220         VERR_RS_RQST_LEN        = 24,
1221         VERR_RS_CMD             = 25,
1222         VERR_RS_CMD_LEN         = 26,
1223         VERR_RS_RCTL            = 27,
1224         VERR_RS_RO              = 28,
1225 };
1226
1227 static char *validation_errors[] = {
1228         "OK",
1229         "Bad CR_ASSOC Length",
1230         "Bad CR_ASSOC Rqst Length",
1231         "Not CR_ASSOC Cmd",
1232         "Bad CR_ASSOC Cmd Length",
1233         "Bad Ersp Ratio",
1234         "Association Allocation Failed",
1235         "Queue Allocation Failed",
1236         "Bad CR_CONN Length",
1237         "Bad CR_CONN Rqst Length",
1238         "Not Association ID",
1239         "Bad Association ID Length",
1240         "No Association",
1241         "Not Connection ID",
1242         "Bad Connection ID Length",
1243         "No Connection",
1244         "Not CR_CONN Cmd",
1245         "Bad CR_CONN Cmd Length",
1246         "Bad DISCONN Length",
1247         "Bad DISCONN Rqst Length",
1248         "Not DISCONN Cmd",
1249         "Bad DISCONN Cmd Length",
1250         "Bad Disconnect Scope",
1251         "Bad RS Length",
1252         "Bad RS Rqst Length",
1253         "Not RS Cmd",
1254         "Bad RS Cmd Length",
1255         "Bad RS R_CTL",
1256         "Bad RS Relative Offset",
1257 };
1258
1259 static void
1260 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1261                         struct nvmet_fc_ls_iod *iod)
1262 {
1263         struct fcnvme_ls_cr_assoc_rqst *rqst =
1264                                 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1265         struct fcnvme_ls_cr_assoc_acc *acc =
1266                                 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1267         struct nvmet_fc_tgt_queue *queue;
1268         int ret = 0;
1269
1270         memset(acc, 0, sizeof(*acc));
1271
1272         /*
1273          * FC-NVME spec changes. There are initiators sending different
1274          * lengths as padding sizes for Create Association Cmd descriptor
1275          * was incorrect.
1276          * Accept anything of "minimum" length. Assume format per 1.15
1277          * spec (with HOSTID reduced to 16 bytes), ignore how long the
1278          * trailing pad length is.
1279          */
1280         if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1281                 ret = VERR_CR_ASSOC_LEN;
1282         else if (be32_to_cpu(rqst->desc_list_len) <
1283                         FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1284                 ret = VERR_CR_ASSOC_RQST_LEN;
1285         else if (rqst->assoc_cmd.desc_tag !=
1286                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1287                 ret = VERR_CR_ASSOC_CMD;
1288         else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1289                         FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1290                 ret = VERR_CR_ASSOC_CMD_LEN;
1291         else if (!rqst->assoc_cmd.ersp_ratio ||
1292                  (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1293                                 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1294                 ret = VERR_ERSP_RATIO;
1295
1296         else {
1297                 /* new association w/ admin queue */
1298                 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1299                 if (!iod->assoc)
1300                         ret = VERR_ASSOC_ALLOC_FAIL;
1301                 else {
1302                         queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1303                                         be16_to_cpu(rqst->assoc_cmd.sqsize));
1304                         if (!queue) {
1305                                 ret = VERR_QUEUE_ALLOC_FAIL;
1306                                 nvmet_fc_tgt_a_put(iod->assoc);
1307                         }
1308                 }
1309         }
1310
1311         if (ret) {
1312                 dev_err(tgtport->dev,
1313                         "Create Association LS failed: %s\n",
1314                         validation_errors[ret]);
1315                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1316                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1317                                 FCNVME_RJT_RC_LOGIC,
1318                                 FCNVME_RJT_EXP_NONE, 0);
1319                 return;
1320         }
1321
1322         queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1323         atomic_set(&queue->connected, 1);
1324         queue->sqhd = 0;        /* best place to init value */
1325
1326         /* format a response */
1327
1328         iod->lsreq->rsplen = sizeof(*acc);
1329
1330         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1331                         fcnvme_lsdesc_len(
1332                                 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1333                         FCNVME_LS_CREATE_ASSOCIATION);
1334         acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1335         acc->associd.desc_len =
1336                         fcnvme_lsdesc_len(
1337                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1338         acc->associd.association_id =
1339                         cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1340         acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1341         acc->connectid.desc_len =
1342                         fcnvme_lsdesc_len(
1343                                 sizeof(struct fcnvme_lsdesc_conn_id));
1344         acc->connectid.connection_id = acc->associd.association_id;
1345 }
1346
1347 static void
1348 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1349                         struct nvmet_fc_ls_iod *iod)
1350 {
1351         struct fcnvme_ls_cr_conn_rqst *rqst =
1352                                 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1353         struct fcnvme_ls_cr_conn_acc *acc =
1354                                 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1355         struct nvmet_fc_tgt_queue *queue;
1356         int ret = 0;
1357
1358         memset(acc, 0, sizeof(*acc));
1359
1360         if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1361                 ret = VERR_CR_CONN_LEN;
1362         else if (rqst->desc_list_len !=
1363                         fcnvme_lsdesc_len(
1364                                 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1365                 ret = VERR_CR_CONN_RQST_LEN;
1366         else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1367                 ret = VERR_ASSOC_ID;
1368         else if (rqst->associd.desc_len !=
1369                         fcnvme_lsdesc_len(
1370                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1371                 ret = VERR_ASSOC_ID_LEN;
1372         else if (rqst->connect_cmd.desc_tag !=
1373                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1374                 ret = VERR_CR_CONN_CMD;
1375         else if (rqst->connect_cmd.desc_len !=
1376                         fcnvme_lsdesc_len(
1377                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1378                 ret = VERR_CR_CONN_CMD_LEN;
1379         else if (!rqst->connect_cmd.ersp_ratio ||
1380                  (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1381                                 be16_to_cpu(rqst->connect_cmd.sqsize)))
1382                 ret = VERR_ERSP_RATIO;
1383
1384         else {
1385                 /* new io queue */
1386                 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1387                                 be64_to_cpu(rqst->associd.association_id));
1388                 if (!iod->assoc)
1389                         ret = VERR_NO_ASSOC;
1390                 else {
1391                         queue = nvmet_fc_alloc_target_queue(iod->assoc,
1392                                         be16_to_cpu(rqst->connect_cmd.qid),
1393                                         be16_to_cpu(rqst->connect_cmd.sqsize));
1394                         if (!queue)
1395                                 ret = VERR_QUEUE_ALLOC_FAIL;
1396
1397                         /* release get taken in nvmet_fc_find_target_assoc */
1398                         nvmet_fc_tgt_a_put(iod->assoc);
1399                 }
1400         }
1401
1402         if (ret) {
1403                 dev_err(tgtport->dev,
1404                         "Create Connection LS failed: %s\n",
1405                         validation_errors[ret]);
1406                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1407                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1408                                 (ret == VERR_NO_ASSOC) ?
1409                                         FCNVME_RJT_RC_INV_ASSOC :
1410                                         FCNVME_RJT_RC_LOGIC,
1411                                 FCNVME_RJT_EXP_NONE, 0);
1412                 return;
1413         }
1414
1415         queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1416         atomic_set(&queue->connected, 1);
1417         queue->sqhd = 0;        /* best place to init value */
1418
1419         /* format a response */
1420
1421         iod->lsreq->rsplen = sizeof(*acc);
1422
1423         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1424                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1425                         FCNVME_LS_CREATE_CONNECTION);
1426         acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1427         acc->connectid.desc_len =
1428                         fcnvme_lsdesc_len(
1429                                 sizeof(struct fcnvme_lsdesc_conn_id));
1430         acc->connectid.connection_id =
1431                         cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1432                                 be16_to_cpu(rqst->connect_cmd.qid)));
1433 }
1434
1435 static void
1436 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1437                         struct nvmet_fc_ls_iod *iod)
1438 {
1439         struct fcnvme_ls_disconnect_rqst *rqst =
1440                         (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1441         struct fcnvme_ls_disconnect_acc *acc =
1442                         (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1443         struct nvmet_fc_tgt_queue *queue = NULL;
1444         struct nvmet_fc_tgt_assoc *assoc;
1445         int ret = 0;
1446         bool del_assoc = false;
1447
1448         memset(acc, 0, sizeof(*acc));
1449
1450         if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1451                 ret = VERR_DISCONN_LEN;
1452         else if (rqst->desc_list_len !=
1453                         fcnvme_lsdesc_len(
1454                                 sizeof(struct fcnvme_ls_disconnect_rqst)))
1455                 ret = VERR_DISCONN_RQST_LEN;
1456         else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1457                 ret = VERR_ASSOC_ID;
1458         else if (rqst->associd.desc_len !=
1459                         fcnvme_lsdesc_len(
1460                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1461                 ret = VERR_ASSOC_ID_LEN;
1462         else if (rqst->discon_cmd.desc_tag !=
1463                         cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1464                 ret = VERR_DISCONN_CMD;
1465         else if (rqst->discon_cmd.desc_len !=
1466                         fcnvme_lsdesc_len(
1467                                 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1468                 ret = VERR_DISCONN_CMD_LEN;
1469         else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1470                         (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1471                 ret = VERR_DISCONN_SCOPE;
1472         else {
1473                 /* match an active association */
1474                 assoc = nvmet_fc_find_target_assoc(tgtport,
1475                                 be64_to_cpu(rqst->associd.association_id));
1476                 iod->assoc = assoc;
1477                 if (assoc) {
1478                         if (rqst->discon_cmd.scope ==
1479                                         FCNVME_DISCONN_CONNECTION) {
1480                                 queue = nvmet_fc_find_target_queue(tgtport,
1481                                                 be64_to_cpu(
1482                                                         rqst->discon_cmd.id));
1483                                 if (!queue) {
1484                                         nvmet_fc_tgt_a_put(assoc);
1485                                         ret = VERR_NO_CONN;
1486                                 }
1487                         }
1488                 } else
1489                         ret = VERR_NO_ASSOC;
1490         }
1491
1492         if (ret) {
1493                 dev_err(tgtport->dev,
1494                         "Disconnect LS failed: %s\n",
1495                         validation_errors[ret]);
1496                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1497                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1498                                 (ret == VERR_NO_ASSOC) ?
1499                                         FCNVME_RJT_RC_INV_ASSOC :
1500                                         (ret == VERR_NO_CONN) ?
1501                                                 FCNVME_RJT_RC_INV_CONN :
1502                                                 FCNVME_RJT_RC_LOGIC,
1503                                 FCNVME_RJT_EXP_NONE, 0);
1504                 return;
1505         }
1506
1507         /* format a response */
1508
1509         iod->lsreq->rsplen = sizeof(*acc);
1510
1511         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1512                         fcnvme_lsdesc_len(
1513                                 sizeof(struct fcnvme_ls_disconnect_acc)),
1514                         FCNVME_LS_DISCONNECT);
1515
1516
1517         /* are we to delete a Connection ID (queue) */
1518         if (queue) {
1519                 int qid = queue->qid;
1520
1521                 nvmet_fc_delete_target_queue(queue);
1522
1523                 /* release the get taken by find_target_queue */
1524                 nvmet_fc_tgt_q_put(queue);
1525
1526                 /* tear association down if io queue terminated */
1527                 if (!qid)
1528                         del_assoc = true;
1529         }
1530
1531         /* release get taken in nvmet_fc_find_target_assoc */
1532         nvmet_fc_tgt_a_put(iod->assoc);
1533
1534         if (del_assoc)
1535                 nvmet_fc_delete_target_assoc(iod->assoc);
1536 }
1537
1538
1539 /* *********************** NVME Ctrl Routines **************************** */
1540
1541
1542 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1543
1544 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1545
1546 static void
1547 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1548 {
1549         struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1550         struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1551
1552         fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1553                                 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1554         nvmet_fc_free_ls_iod(tgtport, iod);
1555         nvmet_fc_tgtport_put(tgtport);
1556 }
1557
1558 static void
1559 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1560                                 struct nvmet_fc_ls_iod *iod)
1561 {
1562         int ret;
1563
1564         fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1565                                   NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1566
1567         ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1568         if (ret)
1569                 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1570 }
1571
1572 /*
1573  * Actual processing routine for received FC-NVME LS Requests from the LLD
1574  */
1575 static void
1576 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1577                         struct nvmet_fc_ls_iod *iod)
1578 {
1579         struct fcnvme_ls_rqst_w0 *w0 =
1580                         (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1581
1582         iod->lsreq->nvmet_fc_private = iod;
1583         iod->lsreq->rspbuf = iod->rspbuf;
1584         iod->lsreq->rspdma = iod->rspdma;
1585         iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1586         /* Be preventative. handlers will later set to valid length */
1587         iod->lsreq->rsplen = 0;
1588
1589         iod->assoc = NULL;
1590
1591         /*
1592          * handlers:
1593          *   parse request input, execute the request, and format the
1594          *   LS response
1595          */
1596         switch (w0->ls_cmd) {
1597         case FCNVME_LS_CREATE_ASSOCIATION:
1598                 /* Creates Association and initial Admin Queue/Connection */
1599                 nvmet_fc_ls_create_association(tgtport, iod);
1600                 break;
1601         case FCNVME_LS_CREATE_CONNECTION:
1602                 /* Creates an IO Queue/Connection */
1603                 nvmet_fc_ls_create_connection(tgtport, iod);
1604                 break;
1605         case FCNVME_LS_DISCONNECT:
1606                 /* Terminate a Queue/Connection or the Association */
1607                 nvmet_fc_ls_disconnect(tgtport, iod);
1608                 break;
1609         default:
1610                 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1611                                 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1612                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1613         }
1614
1615         nvmet_fc_xmt_ls_rsp(tgtport, iod);
1616 }
1617
1618 /*
1619  * Actual processing routine for received FC-NVME LS Requests from the LLD
1620  */
1621 static void
1622 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1623 {
1624         struct nvmet_fc_ls_iod *iod =
1625                 container_of(work, struct nvmet_fc_ls_iod, work);
1626         struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1627
1628         nvmet_fc_handle_ls_rqst(tgtport, iod);
1629 }
1630
1631
1632 /**
1633  * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1634  *                       upon the reception of a NVME LS request.
1635  *
1636  * The nvmet-fc layer will copy payload to an internal structure for
1637  * processing.  As such, upon completion of the routine, the LLDD may
1638  * immediately free/reuse the LS request buffer passed in the call.
1639  *
1640  * If this routine returns error, the LLDD should abort the exchange.
1641  *
1642  * @tgtport:    pointer to the (registered) target port the LS was
1643  *              received on.
1644  * @lsreq:      pointer to a lsreq request structure to be used to reference
1645  *              the exchange corresponding to the LS.
1646  * @lsreqbuf:   pointer to the buffer containing the LS Request
1647  * @lsreqbuf_len: length, in bytes, of the received LS request
1648  */
1649 int
1650 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1651                         struct nvmefc_tgt_ls_req *lsreq,
1652                         void *lsreqbuf, u32 lsreqbuf_len)
1653 {
1654         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1655         struct nvmet_fc_ls_iod *iod;
1656
1657         if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1658                 return -E2BIG;
1659
1660         if (!nvmet_fc_tgtport_get(tgtport))
1661                 return -ESHUTDOWN;
1662
1663         iod = nvmet_fc_alloc_ls_iod(tgtport);
1664         if (!iod) {
1665                 nvmet_fc_tgtport_put(tgtport);
1666                 return -ENOENT;
1667         }
1668
1669         iod->lsreq = lsreq;
1670         iod->fcpreq = NULL;
1671         memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1672         iod->rqstdatalen = lsreqbuf_len;
1673
1674         schedule_work(&iod->work);
1675
1676         return 0;
1677 }
1678 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1679
1680
1681 /*
1682  * **********************
1683  * Start of FCP handling
1684  * **********************
1685  */
1686
1687 static int
1688 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1689 {
1690         struct scatterlist *sg;
1691         struct page *page;
1692         unsigned int nent;
1693         u32 page_len, length;
1694         int i = 0;
1695
1696         length = fod->total_length;
1697         nent = DIV_ROUND_UP(length, PAGE_SIZE);
1698         sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL);
1699         if (!sg)
1700                 goto out;
1701
1702         sg_init_table(sg, nent);
1703
1704         while (length) {
1705                 page_len = min_t(u32, length, PAGE_SIZE);
1706
1707                 page = alloc_page(GFP_KERNEL);
1708                 if (!page)
1709                         goto out_free_pages;
1710
1711                 sg_set_page(&sg[i], page, page_len, 0);
1712                 length -= page_len;
1713                 i++;
1714         }
1715
1716         fod->data_sg = sg;
1717         fod->data_sg_cnt = nent;
1718         fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1719                                 ((fod->io_dir == NVMET_FCP_WRITE) ?
1720                                         DMA_FROM_DEVICE : DMA_TO_DEVICE));
1721                                 /* note: write from initiator perspective */
1722         fod->next_sg = fod->data_sg;
1723
1724         return 0;
1725
1726 out_free_pages:
1727         while (i > 0) {
1728                 i--;
1729                 __free_page(sg_page(&sg[i]));
1730         }
1731         kfree(sg);
1732         fod->data_sg = NULL;
1733         fod->data_sg_cnt = 0;
1734 out:
1735         return NVME_SC_INTERNAL;
1736 }
1737
1738 static void
1739 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1740 {
1741         struct scatterlist *sg;
1742         int count;
1743
1744         if (!fod->data_sg || !fod->data_sg_cnt)
1745                 return;
1746
1747         fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1748                                 ((fod->io_dir == NVMET_FCP_WRITE) ?
1749                                         DMA_FROM_DEVICE : DMA_TO_DEVICE));
1750         for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count)
1751                 __free_page(sg_page(sg));
1752         kfree(fod->data_sg);
1753         fod->data_sg = NULL;
1754         fod->data_sg_cnt = 0;
1755 }
1756
1757
1758 static bool
1759 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1760 {
1761         u32 sqtail, used;
1762
1763         /* egad, this is ugly. And sqtail is just a best guess */
1764         sqtail = atomic_read(&q->sqtail) % q->sqsize;
1765
1766         used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1767         return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1768 }
1769
1770 /*
1771  * Prep RSP payload.
1772  * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
1773  */
1774 static void
1775 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1776                                 struct nvmet_fc_fcp_iod *fod)
1777 {
1778         struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1779         struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1780         struct nvme_completion *cqe = &ersp->cqe;
1781         u32 *cqewd = (u32 *)cqe;
1782         bool send_ersp = false;
1783         u32 rsn, rspcnt, xfr_length;
1784
1785         if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1786                 xfr_length = fod->total_length;
1787         else
1788                 xfr_length = fod->offset;
1789
1790         /*
1791          * check to see if we can send a 0's rsp.
1792          *   Note: to send a 0's response, the NVME-FC host transport will
1793          *   recreate the CQE. The host transport knows: sq id, SQHD (last
1794          *   seen in an ersp), and command_id. Thus it will create a
1795          *   zero-filled CQE with those known fields filled in. Transport
1796          *   must send an ersp for any condition where the cqe won't match
1797          *   this.
1798          *
1799          * Here are the FC-NVME mandated cases where we must send an ersp:
1800          *  every N responses, where N=ersp_ratio
1801          *  force fabric commands to send ersp's (not in FC-NVME but good
1802          *    practice)
1803          *  normal cmds: any time status is non-zero, or status is zero
1804          *     but words 0 or 1 are non-zero.
1805          *  the SQ is 90% or more full
1806          *  the cmd is a fused command
1807          *  transferred data length not equal to cmd iu length
1808          */
1809         rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1810         if (!(rspcnt % fod->queue->ersp_ratio) ||
1811             sqe->opcode == nvme_fabrics_command ||
1812             xfr_length != fod->total_length ||
1813             (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1814             (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1815             queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1816                 send_ersp = true;
1817
1818         /* re-set the fields */
1819         fod->fcpreq->rspaddr = ersp;
1820         fod->fcpreq->rspdma = fod->rspdma;
1821
1822         if (!send_ersp) {
1823                 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1824                 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1825         } else {
1826                 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1827                 rsn = atomic_inc_return(&fod->queue->rsn);
1828                 ersp->rsn = cpu_to_be32(rsn);
1829                 ersp->xfrd_len = cpu_to_be32(xfr_length);
1830                 fod->fcpreq->rsplen = sizeof(*ersp);
1831         }
1832
1833         fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1834                                   sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1835 }
1836
1837 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1838
1839 static void
1840 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1841                                 struct nvmet_fc_fcp_iod *fod)
1842 {
1843         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1844
1845         /* data no longer needed */
1846         nvmet_fc_free_tgt_pgs(fod);
1847
1848         /*
1849          * if an ABTS was received or we issued the fcp_abort early
1850          * don't call abort routine again.
1851          */
1852         /* no need to take lock - lock was taken earlier to get here */
1853         if (!fod->aborted)
1854                 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1855
1856         nvmet_fc_free_fcp_iod(fod->queue, fod);
1857 }
1858
1859 static void
1860 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1861                                 struct nvmet_fc_fcp_iod *fod)
1862 {
1863         int ret;
1864
1865         fod->fcpreq->op = NVMET_FCOP_RSP;
1866         fod->fcpreq->timeout = 0;
1867
1868         nvmet_fc_prep_fcp_rsp(tgtport, fod);
1869
1870         ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1871         if (ret)
1872                 nvmet_fc_abort_op(tgtport, fod);
1873 }
1874
1875 static void
1876 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1877                                 struct nvmet_fc_fcp_iod *fod, u8 op)
1878 {
1879         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1880         struct scatterlist *sg = fod->next_sg;
1881         unsigned long flags;
1882         u32 remaininglen = fod->total_length - fod->offset;
1883         u32 tlen = 0;
1884         int ret;
1885
1886         fcpreq->op = op;
1887         fcpreq->offset = fod->offset;
1888         fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1889
1890         /*
1891          * for next sequence:
1892          *  break at a sg element boundary
1893          *  attempt to keep sequence length capped at
1894          *    NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
1895          *    be longer if a single sg element is larger
1896          *    than that amount. This is done to avoid creating
1897          *    a new sg list to use for the tgtport api.
1898          */
1899         fcpreq->sg = sg;
1900         fcpreq->sg_cnt = 0;
1901         while (tlen < remaininglen &&
1902                fcpreq->sg_cnt < tgtport->max_sg_cnt &&
1903                tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
1904                 fcpreq->sg_cnt++;
1905                 tlen += sg_dma_len(sg);
1906                 sg = sg_next(sg);
1907         }
1908         if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
1909                 fcpreq->sg_cnt++;
1910                 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
1911                 sg = sg_next(sg);
1912         }
1913         if (tlen < remaininglen)
1914                 fod->next_sg = sg;
1915         else
1916                 fod->next_sg = NULL;
1917
1918         fcpreq->transfer_length = tlen;
1919         fcpreq->transferred_length = 0;
1920         fcpreq->fcp_error = 0;
1921         fcpreq->rsplen = 0;
1922
1923         /*
1924          * If the last READDATA request: check if LLDD supports
1925          * combined xfr with response.
1926          */
1927         if ((op == NVMET_FCOP_READDATA) &&
1928             ((fod->offset + fcpreq->transfer_length) == fod->total_length) &&
1929             (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1930                 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1931                 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1932         }
1933
1934         ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1935         if (ret) {
1936                 /*
1937                  * should be ok to set w/o lock as its in the thread of
1938                  * execution (not an async timer routine) and doesn't
1939                  * contend with any clearing action
1940                  */
1941                 fod->abort = true;
1942
1943                 if (op == NVMET_FCOP_WRITEDATA) {
1944                         spin_lock_irqsave(&fod->flock, flags);
1945                         fod->writedataactive = false;
1946                         spin_unlock_irqrestore(&fod->flock, flags);
1947                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1948                 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
1949                         fcpreq->fcp_error = ret;
1950                         fcpreq->transferred_length = 0;
1951                         nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1952                 }
1953         }
1954 }
1955
1956 static inline bool
1957 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
1958 {
1959         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1960         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1961
1962         /* if in the middle of an io and we need to tear down */
1963         if (abort) {
1964                 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
1965                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1966                         return true;
1967                 }
1968
1969                 nvmet_fc_abort_op(tgtport, fod);
1970                 return true;
1971         }
1972
1973         return false;
1974 }
1975
1976 /*
1977  * actual done handler for FCP operations when completed by the lldd
1978  */
1979 static void
1980 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
1981 {
1982         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1983         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1984         unsigned long flags;
1985         bool abort;
1986
1987         spin_lock_irqsave(&fod->flock, flags);
1988         abort = fod->abort;
1989         fod->writedataactive = false;
1990         spin_unlock_irqrestore(&fod->flock, flags);
1991
1992         switch (fcpreq->op) {
1993
1994         case NVMET_FCOP_WRITEDATA:
1995                 if (__nvmet_fc_fod_op_abort(fod, abort))
1996                         return;
1997                 if (fcpreq->fcp_error ||
1998                     fcpreq->transferred_length != fcpreq->transfer_length) {
1999                         spin_lock_irqsave(&fod->flock, flags);
2000                         fod->abort = true;
2001                         spin_unlock_irqrestore(&fod->flock, flags);
2002
2003                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2004                         return;
2005                 }
2006
2007                 fod->offset += fcpreq->transferred_length;
2008                 if (fod->offset != fod->total_length) {
2009                         spin_lock_irqsave(&fod->flock, flags);
2010                         fod->writedataactive = true;
2011                         spin_unlock_irqrestore(&fod->flock, flags);
2012
2013                         /* transfer the next chunk */
2014                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2015                                                 NVMET_FCOP_WRITEDATA);
2016                         return;
2017                 }
2018
2019                 /* data transfer complete, resume with nvmet layer */
2020
2021                 fod->req.execute(&fod->req);
2022
2023                 break;
2024
2025         case NVMET_FCOP_READDATA:
2026         case NVMET_FCOP_READDATA_RSP:
2027                 if (__nvmet_fc_fod_op_abort(fod, abort))
2028                         return;
2029                 if (fcpreq->fcp_error ||
2030                     fcpreq->transferred_length != fcpreq->transfer_length) {
2031                         nvmet_fc_abort_op(tgtport, fod);
2032                         return;
2033                 }
2034
2035                 /* success */
2036
2037                 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2038                         /* data no longer needed */
2039                         nvmet_fc_free_tgt_pgs(fod);
2040                         nvmet_fc_free_fcp_iod(fod->queue, fod);
2041                         return;
2042                 }
2043
2044                 fod->offset += fcpreq->transferred_length;
2045                 if (fod->offset != fod->total_length) {
2046                         /* transfer the next chunk */
2047                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2048                                                 NVMET_FCOP_READDATA);
2049                         return;
2050                 }
2051
2052                 /* data transfer complete, send response */
2053
2054                 /* data no longer needed */
2055                 nvmet_fc_free_tgt_pgs(fod);
2056
2057                 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2058
2059                 break;
2060
2061         case NVMET_FCOP_RSP:
2062                 if (__nvmet_fc_fod_op_abort(fod, abort))
2063                         return;
2064                 nvmet_fc_free_fcp_iod(fod->queue, fod);
2065                 break;
2066
2067         default:
2068                 break;
2069         }
2070 }
2071
2072 static void
2073 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work)
2074 {
2075         struct nvmet_fc_fcp_iod *fod =
2076                 container_of(work, struct nvmet_fc_fcp_iod, done_work);
2077
2078         nvmet_fc_fod_op_done(fod);
2079 }
2080
2081 static void
2082 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2083 {
2084         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2085         struct nvmet_fc_tgt_queue *queue = fod->queue;
2086
2087         if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR)
2088                 /* context switch so completion is not in ISR context */
2089                 queue_work_on(queue->cpu, queue->work_q, &fod->done_work);
2090         else
2091                 nvmet_fc_fod_op_done(fod);
2092 }
2093
2094 /*
2095  * actual completion handler after execution by the nvmet layer
2096  */
2097 static void
2098 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2099                         struct nvmet_fc_fcp_iod *fod, int status)
2100 {
2101         struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2102         struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2103         unsigned long flags;
2104         bool abort;
2105
2106         spin_lock_irqsave(&fod->flock, flags);
2107         abort = fod->abort;
2108         spin_unlock_irqrestore(&fod->flock, flags);
2109
2110         /* if we have a CQE, snoop the last sq_head value */
2111         if (!status)
2112                 fod->queue->sqhd = cqe->sq_head;
2113
2114         if (abort) {
2115                 nvmet_fc_abort_op(tgtport, fod);
2116                 return;
2117         }
2118
2119         /* if an error handling the cmd post initial parsing */
2120         if (status) {
2121                 /* fudge up a failed CQE status for our transport error */
2122                 memset(cqe, 0, sizeof(*cqe));
2123                 cqe->sq_head = fod->queue->sqhd;        /* echo last cqe sqhd */
2124                 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2125                 cqe->command_id = sqe->command_id;
2126                 cqe->status = cpu_to_le16(status);
2127         } else {
2128
2129                 /*
2130                  * try to push the data even if the SQE status is non-zero.
2131                  * There may be a status where data still was intended to
2132                  * be moved
2133                  */
2134                 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2135                         /* push the data over before sending rsp */
2136                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2137                                                 NVMET_FCOP_READDATA);
2138                         return;
2139                 }
2140
2141                 /* writes & no data - fall thru */
2142         }
2143
2144         /* data no longer needed */
2145         nvmet_fc_free_tgt_pgs(fod);
2146
2147         nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2148 }
2149
2150
2151 static void
2152 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2153 {
2154         struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2155         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2156
2157         __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2158 }
2159
2160
2161 /*
2162  * Actual processing routine for received FC-NVME LS Requests from the LLD
2163  */
2164 static void
2165 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2166                         struct nvmet_fc_fcp_iod *fod)
2167 {
2168         struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2169         int ret;
2170
2171         /*
2172          * Fused commands are currently not supported in the linux
2173          * implementation.
2174          *
2175          * As such, the implementation of the FC transport does not
2176          * look at the fused commands and order delivery to the upper
2177          * layer until we have both based on csn.
2178          */
2179
2180         fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2181
2182         fod->total_length = be32_to_cpu(cmdiu->data_len);
2183         if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2184                 fod->io_dir = NVMET_FCP_WRITE;
2185                 if (!nvme_is_write(&cmdiu->sqe))
2186                         goto transport_error;
2187         } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2188                 fod->io_dir = NVMET_FCP_READ;
2189                 if (nvme_is_write(&cmdiu->sqe))
2190                         goto transport_error;
2191         } else {
2192                 fod->io_dir = NVMET_FCP_NODATA;
2193                 if (fod->total_length)
2194                         goto transport_error;
2195         }
2196
2197         fod->req.cmd = &fod->cmdiubuf.sqe;
2198         fod->req.rsp = &fod->rspiubuf.cqe;
2199         fod->req.port = fod->queue->port;
2200
2201         /* ensure nvmet handlers will set cmd handler callback */
2202         fod->req.execute = NULL;
2203
2204         /* clear any response payload */
2205         memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2206
2207         fod->data_sg = NULL;
2208         fod->data_sg_cnt = 0;
2209
2210         ret = nvmet_req_init(&fod->req,
2211                                 &fod->queue->nvme_cq,
2212                                 &fod->queue->nvme_sq,
2213                                 &nvmet_fc_tgt_fcp_ops);
2214         if (!ret) {
2215                 /* bad SQE content or invalid ctrl state */
2216                 /* nvmet layer has already called op done to send rsp. */
2217                 return;
2218         }
2219
2220         /* keep a running counter of tail position */
2221         atomic_inc(&fod->queue->sqtail);
2222
2223         if (fod->total_length) {
2224                 ret = nvmet_fc_alloc_tgt_pgs(fod);
2225                 if (ret) {
2226                         nvmet_req_complete(&fod->req, ret);
2227                         return;
2228                 }
2229         }
2230         fod->req.sg = fod->data_sg;
2231         fod->req.sg_cnt = fod->data_sg_cnt;
2232         fod->offset = 0;
2233
2234         if (fod->io_dir == NVMET_FCP_WRITE) {
2235                 /* pull the data over before invoking nvmet layer */
2236                 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2237                 return;
2238         }
2239
2240         /*
2241          * Reads or no data:
2242          *
2243          * can invoke the nvmet_layer now. If read data, cmd completion will
2244          * push the data
2245          */
2246
2247         fod->req.execute(&fod->req);
2248
2249         return;
2250
2251 transport_error:
2252         nvmet_fc_abort_op(tgtport, fod);
2253 }
2254
2255 /*
2256  * Actual processing routine for received FC-NVME LS Requests from the LLD
2257  */
2258 static void
2259 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
2260 {
2261         struct nvmet_fc_fcp_iod *fod =
2262                 container_of(work, struct nvmet_fc_fcp_iod, work);
2263         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2264
2265         nvmet_fc_handle_fcp_rqst(tgtport, fod);
2266 }
2267
2268 /**
2269  * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2270  *                       upon the reception of a NVME FCP CMD IU.
2271  *
2272  * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2273  * layer for processing.
2274  *
2275  * The nvmet_fc layer allocates a local job structure (struct
2276  * nvmet_fc_fcp_iod) from the queue for the io and copies the
2277  * CMD IU buffer to the job structure. As such, on a successful
2278  * completion (returns 0), the LLDD may immediately free/reuse
2279  * the CMD IU buffer passed in the call.
2280  *
2281  * However, in some circumstances, due to the packetized nature of FC
2282  * and the api of the FC LLDD which may issue a hw command to send the
2283  * response, but the LLDD may not get the hw completion for that command
2284  * and upcall the nvmet_fc layer before a new command may be
2285  * asynchronously received - its possible for a command to be received
2286  * before the LLDD and nvmet_fc have recycled the job structure. It gives
2287  * the appearance of more commands received than fits in the sq.
2288  * To alleviate this scenario, a temporary queue is maintained in the
2289  * transport for pending LLDD requests waiting for a queue job structure.
2290  * In these "overrun" cases, a temporary queue element is allocated
2291  * the LLDD request and CMD iu buffer information remembered, and the
2292  * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2293  * structure is freed, it is immediately reallocated for anything on the
2294  * pending request list. The LLDDs defer_rcv() callback is called,
2295  * informing the LLDD that it may reuse the CMD IU buffer, and the io
2296  * is then started normally with the transport.
2297  *
2298  * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2299  * the completion as successful but must not reuse the CMD IU buffer
2300  * until the LLDD's defer_rcv() callback has been called for the
2301  * corresponding struct nvmefc_tgt_fcp_req pointer.
2302  *
2303  * If there is any other condition in which an error occurs, the
2304  * transport will return a non-zero status indicating the error.
2305  * In all cases other than -EOVERFLOW, the transport has not accepted the
2306  * request and the LLDD should abort the exchange.
2307  *
2308  * @target_port: pointer to the (registered) target port the FCP CMD IU
2309  *              was received on.
2310  * @fcpreq:     pointer to a fcpreq request structure to be used to reference
2311  *              the exchange corresponding to the FCP Exchange.
2312  * @cmdiubuf:   pointer to the buffer containing the FCP CMD IU
2313  * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2314  */
2315 int
2316 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2317                         struct nvmefc_tgt_fcp_req *fcpreq,
2318                         void *cmdiubuf, u32 cmdiubuf_len)
2319 {
2320         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2321         struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2322         struct nvmet_fc_tgt_queue *queue;
2323         struct nvmet_fc_fcp_iod *fod;
2324         struct nvmet_fc_defer_fcp_req *deferfcp;
2325         unsigned long flags;
2326
2327         /* validate iu, so the connection id can be used to find the queue */
2328         if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2329                         (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2330                         (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2331                         (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2332                 return -EIO;
2333
2334         queue = nvmet_fc_find_target_queue(tgtport,
2335                                 be64_to_cpu(cmdiu->connection_id));
2336         if (!queue)
2337                 return -ENOTCONN;
2338
2339         /*
2340          * note: reference taken by find_target_queue
2341          * After successful fod allocation, the fod will inherit the
2342          * ownership of that reference and will remove the reference
2343          * when the fod is freed.
2344          */
2345
2346         spin_lock_irqsave(&queue->qlock, flags);
2347
2348         fod = nvmet_fc_alloc_fcp_iod(queue);
2349         if (fod) {
2350                 spin_unlock_irqrestore(&queue->qlock, flags);
2351
2352                 fcpreq->nvmet_fc_private = fod;
2353                 fod->fcpreq = fcpreq;
2354
2355                 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2356
2357                 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2358
2359                 return 0;
2360         }
2361
2362         if (!tgtport->ops->defer_rcv) {
2363                 spin_unlock_irqrestore(&queue->qlock, flags);
2364                 /* release the queue lookup reference */
2365                 nvmet_fc_tgt_q_put(queue);
2366                 return -ENOENT;
2367         }
2368
2369         deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2370                         struct nvmet_fc_defer_fcp_req, req_list);
2371         if (deferfcp) {
2372                 /* Just re-use one that was previously allocated */
2373                 list_del(&deferfcp->req_list);
2374         } else {
2375                 spin_unlock_irqrestore(&queue->qlock, flags);
2376
2377                 /* Now we need to dynamically allocate one */
2378                 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2379                 if (!deferfcp) {
2380                         /* release the queue lookup reference */
2381                         nvmet_fc_tgt_q_put(queue);
2382                         return -ENOMEM;
2383                 }
2384                 spin_lock_irqsave(&queue->qlock, flags);
2385         }
2386
2387         /* For now, use rspaddr / rsplen to save payload information */
2388         fcpreq->rspaddr = cmdiubuf;
2389         fcpreq->rsplen  = cmdiubuf_len;
2390         deferfcp->fcp_req = fcpreq;
2391
2392         /* defer processing till a fod becomes available */
2393         list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2394
2395         /* NOTE: the queue lookup reference is still valid */
2396
2397         spin_unlock_irqrestore(&queue->qlock, flags);
2398
2399         return -EOVERFLOW;
2400 }
2401 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2402
2403 /**
2404  * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2405  *                       upon the reception of an ABTS for a FCP command
2406  *
2407  * Notify the transport that an ABTS has been received for a FCP command
2408  * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2409  * LLDD believes the command is still being worked on
2410  * (template_ops->fcp_req_release() has not been called).
2411  *
2412  * The transport will wait for any outstanding work (an op to the LLDD,
2413  * which the lldd should complete with error due to the ABTS; or the
2414  * completion from the nvmet layer of the nvme command), then will
2415  * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2416  * return the i/o context to the LLDD.  The LLDD may send the BA_ACC
2417  * to the ABTS either after return from this function (assuming any
2418  * outstanding op work has been terminated) or upon the callback being
2419  * called.
2420  *
2421  * @target_port: pointer to the (registered) target port the FCP CMD IU
2422  *              was received on.
2423  * @fcpreq:     pointer to the fcpreq request structure that corresponds
2424  *              to the exchange that received the ABTS.
2425  */
2426 void
2427 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2428                         struct nvmefc_tgt_fcp_req *fcpreq)
2429 {
2430         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2431         struct nvmet_fc_tgt_queue *queue;
2432         unsigned long flags;
2433
2434         if (!fod || fod->fcpreq != fcpreq)
2435                 /* job appears to have already completed, ignore abort */
2436                 return;
2437
2438         queue = fod->queue;
2439
2440         spin_lock_irqsave(&queue->qlock, flags);
2441         if (fod->active) {
2442                 /*
2443                  * mark as abort. The abort handler, invoked upon completion
2444                  * of any work, will detect the aborted status and do the
2445                  * callback.
2446                  */
2447                 spin_lock(&fod->flock);
2448                 fod->abort = true;
2449                 fod->aborted = true;
2450                 spin_unlock(&fod->flock);
2451         }
2452         spin_unlock_irqrestore(&queue->qlock, flags);
2453 }
2454 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2455
2456
2457 struct nvmet_fc_traddr {
2458         u64     nn;
2459         u64     pn;
2460 };
2461
2462 static int
2463 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2464 {
2465         u64 token64;
2466
2467         if (match_u64(sstr, &token64))
2468                 return -EINVAL;
2469         *val = token64;
2470
2471         return 0;
2472 }
2473
2474 /*
2475  * This routine validates and extracts the WWN's from the TRADDR string.
2476  * As kernel parsers need the 0x to determine number base, universally
2477  * build string to parse with 0x prefix before parsing name strings.
2478  */
2479 static int
2480 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2481 {
2482         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2483         substring_t wwn = { name, &name[sizeof(name)-1] };
2484         int nnoffset, pnoffset;
2485
2486         /* validate it string one of the 2 allowed formats */
2487         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2488                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2489                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2490                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2491                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2492                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2493                                                 NVME_FC_TRADDR_OXNNLEN;
2494         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2495                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2496                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2497                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
2498                 nnoffset = NVME_FC_TRADDR_NNLEN;
2499                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2500         } else
2501                 goto out_einval;
2502
2503         name[0] = '0';
2504         name[1] = 'x';
2505         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2506
2507         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2508         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2509                 goto out_einval;
2510
2511         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2512         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2513                 goto out_einval;
2514
2515         return 0;
2516
2517 out_einval:
2518         pr_warn("%s: bad traddr string\n", __func__);
2519         return -EINVAL;
2520 }
2521
2522 static int
2523 nvmet_fc_add_port(struct nvmet_port *port)
2524 {
2525         struct nvmet_fc_tgtport *tgtport;
2526         struct nvmet_fc_traddr traddr = { 0L, 0L };
2527         unsigned long flags;
2528         int ret;
2529
2530         /* validate the address info */
2531         if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2532             (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2533                 return -EINVAL;
2534
2535         /* map the traddr address info to a target port */
2536
2537         ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2538                         sizeof(port->disc_addr.traddr));
2539         if (ret)
2540                 return ret;
2541
2542         ret = -ENXIO;
2543         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2544         list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2545                 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2546                     (tgtport->fc_target_port.port_name == traddr.pn)) {
2547                         /* a FC port can only be 1 nvmet port id */
2548                         if (!tgtport->port) {
2549                                 tgtport->port = port;
2550                                 port->priv = tgtport;
2551                                 nvmet_fc_tgtport_get(tgtport);
2552                                 ret = 0;
2553                         } else
2554                                 ret = -EALREADY;
2555                         break;
2556                 }
2557         }
2558         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2559         return ret;
2560 }
2561
2562 static void
2563 nvmet_fc_remove_port(struct nvmet_port *port)
2564 {
2565         struct nvmet_fc_tgtport *tgtport = port->priv;
2566         unsigned long flags;
2567         bool matched = false;
2568
2569         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2570         if (tgtport->port == port) {
2571                 matched = true;
2572                 tgtport->port = NULL;
2573         }
2574         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2575
2576         if (matched)
2577                 nvmet_fc_tgtport_put(tgtport);
2578 }
2579
2580 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2581         .owner                  = THIS_MODULE,
2582         .type                   = NVMF_TRTYPE_FC,
2583         .msdbd                  = 1,
2584         .add_port               = nvmet_fc_add_port,
2585         .remove_port            = nvmet_fc_remove_port,
2586         .queue_response         = nvmet_fc_fcp_nvme_cmd_done,
2587         .delete_ctrl            = nvmet_fc_delete_ctrl,
2588 };
2589
2590 static int __init nvmet_fc_init_module(void)
2591 {
2592         return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2593 }
2594
2595 static void __exit nvmet_fc_exit_module(void)
2596 {
2597         /* sanity check - all lports should be removed */
2598         if (!list_empty(&nvmet_fc_target_list))
2599                 pr_warn("%s: targetport list not empty\n", __func__);
2600
2601         nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2602
2603         ida_destroy(&nvmet_fc_tgtport_cnt);
2604 }
2605
2606 module_init(nvmet_fc_init_module);
2607 module_exit(nvmet_fc_exit_module);
2608
2609 MODULE_LICENSE("GPL v2");