drivers/nvme/host/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics TCP host.
   4  * Copyright (c) 2018 Lightbits Labs. All rights reserved.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/slab.h>
  10 #include <linux/err.h>
  11 #include <linux/nvme-tcp.h>
  12 #include <net/sock.h>
  13 #include <net/tcp.h>
  14 #include <linux/blk-mq.h>
  15 #include <crypto/hash.h>
  16 #include <net/busy_poll.h>
  17
  18 #include "nvme.h"
  19 #include "fabrics.h"
  20
  21 struct nvme_tcp_queue;
  22
  23 /* Define the socket priority to use for connections were it is desirable
  24  * that the NIC consider performing optimized packet processing or filtering.
  25  * A non-zero value being sufficient to indicate general consideration of any
  26  * possible optimization.  Making it a module param allows for alternative
  27  * values that may be unique for some NIC implementations.
  28  */
  29 static int so_priority;
  30 module_param(so_priority, int, 0644);
  31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
  32
  33 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  34 /* lockdep can detect a circular dependency of the form
  35  *   sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
  36  * because dependencies are tracked for both nvme-tcp and user contexts. Using
  37  * a separate class prevents lockdep from conflating nvme-tcp socket use with
  38  * user-space socket API use.
  39  */
  40 static struct lock_class_key nvme_tcp_sk_key[2];
  41 static struct lock_class_key nvme_tcp_slock_key[2];
  42
  43 static void nvme_tcp_reclassify_socket(struct socket *sock)
  44 {
  45         struct sock *sk = sock->sk;
  46
  47         if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
  48                 return;
  49
  50         switch (sk->sk_family) {
  51         case AF_INET:
  52                 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
  53                                               &nvme_tcp_slock_key[0],
  54                                               "sk_lock-AF_INET-NVME",
  55                                               &nvme_tcp_sk_key[0]);
  56                 break;
  57         case AF_INET6:
  58                 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
  59                                               &nvme_tcp_slock_key[1],
  60                                               "sk_lock-AF_INET6-NVME",
  61                                               &nvme_tcp_sk_key[1]);
  62                 break;
  63         default:
  64                 WARN_ON_ONCE(1);
  65         }
  66 }
  67 #else
  68 static void nvme_tcp_reclassify_socket(struct socket *sock) { }
  69 #endif
  70
  71 enum nvme_tcp_send_state {
  72         NVME_TCP_SEND_CMD_PDU = 0,
  73         NVME_TCP_SEND_H2C_PDU,
  74         NVME_TCP_SEND_DATA,
  75         NVME_TCP_SEND_DDGST,
  76 };
  77
  78 struct nvme_tcp_request {
  79         struct nvme_request     req;
  80         void                    *pdu;
  81         struct nvme_tcp_queue   *queue;
  82         u32                     data_len;
  83         u32                     pdu_len;
  84         u32                     pdu_sent;
  85         u16                     ttag;
  86         struct list_head        entry;
  87         struct llist_node       lentry;
  88         __le32                  ddgst;
  89
  90         struct bio              *curr_bio;
  91         struct iov_iter         iter;
  92
  93         /* send state */
  94         size_t                  offset;
  95         size_t                  data_sent;
  96         enum nvme_tcp_send_state state;
  97 };
  98
  99 enum nvme_tcp_queue_flags {
 100         NVME_TCP_Q_ALLOCATED    = 0,
 101         NVME_TCP_Q_LIVE         = 1,
 102         NVME_TCP_Q_POLLING      = 2,
 103 };
 104
 105 enum nvme_tcp_recv_state {
 106         NVME_TCP_RECV_PDU = 0,
 107         NVME_TCP_RECV_DATA,
 108         NVME_TCP_RECV_DDGST,
 109 };
 110
 111 struct nvme_tcp_ctrl;
 112 struct nvme_tcp_queue {
 113         struct socket           *sock;
 114         struct work_struct      io_work;
 115         int                     io_cpu;
 116
 117         struct mutex            queue_lock;
 118         struct mutex            send_mutex;
 119         struct llist_head       req_list;
 120         struct list_head        send_list;
 121
 122         /* recv state */
 123         void                    *pdu;
 124         int                     pdu_remaining;
 125         int                     pdu_offset;
 126         size_t                  data_remaining;
 127         size_t                  ddgst_remaining;
 128         unsigned int            nr_cqe;
 129
 130         /* send state */
 131         struct nvme_tcp_request *request;
 132
 133         int                     queue_size;
 134         size_t                  cmnd_capsule_len;
 135         struct nvme_tcp_ctrl    *ctrl;
 136         unsigned long           flags;
 137         bool                    rd_enabled;
 138
 139         bool                    hdr_digest;
 140         bool                    data_digest;
 141         struct ahash_request    *rcv_hash;
 142         struct ahash_request    *snd_hash;
 143         __le32                  exp_ddgst;
 144         __le32                  recv_ddgst;
 145
 146         struct page_frag_cache  pf_cache;
 147
 148         void (*state_change)(struct sock *);
 149         void (*data_ready)(struct sock *);
 150         void (*write_space)(struct sock *);
 151 };
 152
 153 struct nvme_tcp_ctrl {
 154         /* read only in the hot path */
 155         struct nvme_tcp_queue   *queues;
 156         struct blk_mq_tag_set   tag_set;
 157
 158         /* other member variables */
 159         struct list_head        list;
 160         struct blk_mq_tag_set   admin_tag_set;
 161         struct sockaddr_storage addr;
 162         struct sockaddr_storage src_addr;
 163         struct nvme_ctrl        ctrl;
 164
 165         struct work_struct      err_work;
 166         struct delayed_work     connect_work;
 167         struct nvme_tcp_request async_req;
 168         u32                     io_queues[HCTX_MAX_TYPES];
 169 };
 170
 171 static LIST_HEAD(nvme_tcp_ctrl_list);
 172 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 173 static struct workqueue_struct *nvme_tcp_wq;
 174 static const struct blk_mq_ops nvme_tcp_mq_ops;
 175 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 176 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 177
 178 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 179 {
 180         return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
 181 }
 182
 183 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
 184 {
 185         return queue - queue->ctrl->queues;
 186 }
 187
 188 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
 189 {
 190         u32 queue_idx = nvme_tcp_queue_id(queue);
 191
 192         if (queue_idx == 0)
 193                 return queue->ctrl->admin_tag_set.tags[queue_idx];
 194         return queue->ctrl->tag_set.tags[queue_idx - 1];
 195 }
 196
 197 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
 198 {
 199         return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 200 }
 201
 202 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
 203 {
 204         return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 205 }
 206
 207 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
 208 {
 209         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
 210 }
 211
 212 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
 213 {
 214         return req == &req->queue->ctrl->async_req;
 215 }
 216
 217 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
 218 {
 219         struct request *rq;
 220
 221         if (unlikely(nvme_tcp_async_req(req)))
 222                 return false; /* async events don't have a request */
 223
 224         rq = blk_mq_rq_from_pdu(req);
 225
 226         return rq_data_dir(rq) == WRITE && req->data_len &&
 227                 req->data_len <= nvme_tcp_inline_data_size(req->queue);
 228 }
 229
 230 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
 231 {
 232         return req->iter.bvec->bv_page;
 233 }
 234
 235 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
 236 {
 237         return req->iter.bvec->bv_offset + req->iter.iov_offset;
 238 }
 239
 240 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
 241 {
 242         return min_t(size_t, iov_iter_single_seg_count(&req->iter),
 243                         req->pdu_len - req->pdu_sent);
 244 }
 245
 246 static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req)
 247 {
 248         return req->iter.iov_offset;
 249 }
 250
 251 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
 252 {
 253         return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
 254                         req->pdu_len - req->pdu_sent : 0;
 255 }
 256
 257 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
 258                 int len)
 259 {
 260         return nvme_tcp_pdu_data_left(req) <= len;
 261 }
 262
 263 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
 264                 unsigned int dir)
 265 {
 266         struct request *rq = blk_mq_rq_from_pdu(req);
 267         struct bio_vec *vec;
 268         unsigned int size;
 269         int nsegs;
 270         size_t offset;
 271
 272         if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
 273                 vec = &rq->special_vec;
 274                 nsegs = 1;
 275                 size = blk_rq_payload_bytes(rq);
 276                 offset = 0;
 277         } else {
 278                 struct bio *bio = req->curr_bio;
 279
 280                 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 281                 nsegs = bio_segments(bio);
 282                 size = bio->bi_iter.bi_size;
 283                 offset = bio->bi_iter.bi_bvec_done;
 284         }
 285
 286         iov_iter_bvec(&req->iter, dir, vec, nsegs, size);
 287         req->iter.iov_offset = offset;
 288 }
 289
 290 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 291                 int len)
 292 {
 293         req->data_sent += len;
 294         req->pdu_sent += len;
 295         iov_iter_advance(&req->iter, len);
 296         if (!iov_iter_count(&req->iter) &&
 297             req->data_sent < req->data_len) {
 298                 req->curr_bio = req->curr_bio->bi_next;
 299                 nvme_tcp_init_iter(req, WRITE);
 300         }
 301 }
 302
 303 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
 304 {
 305         int ret;
 306
 307         /* drain the send queue as much as we can... */
 308         do {
 309                 ret = nvme_tcp_try_send(queue);
 310         } while (ret > 0);
 311 }
 312
 313 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
 314 {
 315         return !list_empty(&queue->send_list) ||
 316                 !llist_empty(&queue->req_list);
 317 }
 318
 319 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 320                 bool sync, bool last)
 321 {
 322         struct nvme_tcp_queue *queue = req->queue;
 323         bool empty;
 324
 325         empty = llist_add(&req->lentry, &queue->req_list) &&
 326                 list_empty(&queue->send_list) && !queue->request;
 327
 328         /*
 329          * if we're the first on the send_list and we can try to send
 330          * directly, otherwise queue io_work. Also, only do that if we
 331          * are on the same cpu, so we don't introduce contention.
 332          */
 333         if (queue->io_cpu == raw_smp_processor_id() &&
 334             sync && empty && mutex_trylock(&queue->send_mutex)) {
 335                 nvme_tcp_send_all(queue);
 336                 mutex_unlock(&queue->send_mutex);
 337         }
 338
 339         if (last && nvme_tcp_queue_more(queue))
 340                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 341 }
 342
 343 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
 344 {
 345         struct nvme_tcp_request *req;
 346         struct llist_node *node;
 347
 348         for (node = llist_del_all(&queue->req_list); node; node = node->next) {
 349                 req = llist_entry(node, struct nvme_tcp_request, lentry);
 350                 list_add(&req->entry, &queue->send_list);
 351         }
 352 }
 353
 354 static inline struct nvme_tcp_request *
 355 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
 356 {
 357         struct nvme_tcp_request *req;
 358
 359         req = list_first_entry_or_null(&queue->send_list,
 360                         struct nvme_tcp_request, entry);
 361         if (!req) {
 362                 nvme_tcp_process_req_list(queue);
 363                 req = list_first_entry_or_null(&queue->send_list,
 364                                 struct nvme_tcp_request, entry);
 365                 if (unlikely(!req))
 366                         return NULL;
 367         }
 368
 369         list_del(&req->entry);
 370         return req;
 371 }
 372
 373 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
 374                 __le32 *dgst)
 375 {
 376         ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
 377         crypto_ahash_final(hash);
 378 }
 379
 380 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
 381                 struct page *page, off_t off, size_t len)
 382 {
 383         struct scatterlist sg;
 384
 385         sg_init_marker(&sg, 1);
 386         sg_set_page(&sg, page, len, off);
 387         ahash_request_set_crypt(hash, &sg, NULL, len);
 388         crypto_ahash_update(hash);
 389 }
 390
 391 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
 392                 void *pdu, size_t len)
 393 {
 394         struct scatterlist sg;
 395
 396         sg_init_one(&sg, pdu, len);
 397         ahash_request_set_crypt(hash, &sg, pdu + len, len);
 398         crypto_ahash_digest(hash);
 399 }
 400
 401 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
 402                 void *pdu, size_t pdu_len)
 403 {
 404         struct nvme_tcp_hdr *hdr = pdu;
 405         __le32 recv_digest;
 406         __le32 exp_digest;
 407
 408         if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
 409                 dev_err(queue->ctrl->ctrl.device,
 410                         "queue %d: header digest flag is cleared\n",
 411                         nvme_tcp_queue_id(queue));
 412                 return -EPROTO;
 413         }
 414
 415         recv_digest = *(__le32 *)(pdu + hdr->hlen);
 416         nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
 417         exp_digest = *(__le32 *)(pdu + hdr->hlen);
 418         if (recv_digest != exp_digest) {
 419                 dev_err(queue->ctrl->ctrl.device,
 420                         "header digest error: recv %#x expected %#x\n",
 421                         le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
 422                 return -EIO;
 423         }
 424
 425         return 0;
 426 }
 427
 428 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
 429 {
 430         struct nvme_tcp_hdr *hdr = pdu;
 431         u8 digest_len = nvme_tcp_hdgst_len(queue);
 432         u32 len;
 433
 434         len = le32_to_cpu(hdr->plen) - hdr->hlen -
 435                 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
 436
 437         if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
 438                 dev_err(queue->ctrl->ctrl.device,
 439                         "queue %d: data digest flag is cleared\n",
 440                 nvme_tcp_queue_id(queue));
 441                 return -EPROTO;
 442         }
 443         crypto_ahash_init(queue->rcv_hash);
 444
 445         return 0;
 446 }
 447
 448 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
 449                 struct request *rq, unsigned int hctx_idx)
 450 {
 451         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 452
 453         page_frag_free(req->pdu);
 454 }
 455
 456 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 457                 struct request *rq, unsigned int hctx_idx,
 458                 unsigned int numa_node)
 459 {
 460         struct nvme_tcp_ctrl *ctrl = set->driver_data;
 461         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 462         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 463         struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
 464         u8 hdgst = nvme_tcp_hdgst_len(queue);
 465
 466         req->pdu = page_frag_alloc(&queue->pf_cache,
 467                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
 468                 GFP_KERNEL | __GFP_ZERO);
 469         if (!req->pdu)
 470                 return -ENOMEM;
 471
 472         req->queue = queue;
 473         nvme_req(rq)->ctrl = &ctrl->ctrl;
 474
 475         return 0;
 476 }
 477
 478 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 479                 unsigned int hctx_idx)
 480 {
 481         struct nvme_tcp_ctrl *ctrl = data;
 482         struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
 483
 484         hctx->driver_data = queue;
 485         return 0;
 486 }
 487
 488 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 489                 unsigned int hctx_idx)
 490 {
 491         struct nvme_tcp_ctrl *ctrl = data;
 492         struct nvme_tcp_queue *queue = &ctrl->queues[0];
 493
 494         hctx->driver_data = queue;
 495         return 0;
 496 }
 497
 498 static enum nvme_tcp_recv_state
 499 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
 500 {
 501         return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
 502                 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
 503                 NVME_TCP_RECV_DATA;
 504 }
 505
 506 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
 507 {
 508         queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
 509                                 nvme_tcp_hdgst_len(queue);
 510         queue->pdu_offset = 0;
 511         queue->data_remaining = -1;
 512         queue->ddgst_remaining = 0;
 513 }
 514
 515 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
 516 {
 517         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 518                 return;
 519
 520         dev_warn(ctrl->device, "starting error recovery\n");
 521         queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
 522 }
 523
 524 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 525                 struct nvme_completion *cqe)
 526 {
 527         struct request *rq;
 528
 529         rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
 530         if (!rq) {
 531                 dev_err(queue->ctrl->ctrl.device,
 532                         "got bad cqe.command_id %#x on queue %d\n",
 533                         cqe->command_id, nvme_tcp_queue_id(queue));
 534                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 535                 return -EINVAL;
 536         }
 537
 538         if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
 539                 nvme_complete_rq(rq);
 540         queue->nr_cqe++;
 541
 542         return 0;
 543 }
 544
 545 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
 546                 struct nvme_tcp_data_pdu *pdu)
 547 {
 548         struct request *rq;
 549
 550         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 551         if (!rq) {
 552                 dev_err(queue->ctrl->ctrl.device,
 553                         "got bad c2hdata.command_id %#x on queue %d\n",
 554                         pdu->command_id, nvme_tcp_queue_id(queue));
 555                 return -ENOENT;
 556         }
 557
 558         if (!blk_rq_payload_bytes(rq)) {
 559                 dev_err(queue->ctrl->ctrl.device,
 560                         "queue %d tag %#x unexpected data\n",
 561                         nvme_tcp_queue_id(queue), rq->tag);
 562                 return -EIO;
 563         }
 564
 565         queue->data_remaining = le32_to_cpu(pdu->data_length);
 566
 567         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
 568             unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
 569                 dev_err(queue->ctrl->ctrl.device,
 570                         "queue %d tag %#x SUCCESS set but not last PDU\n",
 571                         nvme_tcp_queue_id(queue), rq->tag);
 572                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 573                 return -EPROTO;
 574         }
 575
 576         return 0;
 577 }
 578
 579 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
 580                 struct nvme_tcp_rsp_pdu *pdu)
 581 {
 582         struct nvme_completion *cqe = &pdu->cqe;
 583         int ret = 0;
 584
 585         /*
 586          * AEN requests are special as they don't time out and can
 587          * survive any kind of queue freeze and often don't respond to
 588          * aborts.  We don't even bother to allocate a struct request
 589          * for them but rather special case them here.
 590          */
 591         if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
 592                                      cqe->command_id)))
 593                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 594                                 &cqe->result);
 595         else
 596                 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
 597
 598         return ret;
 599 }
 600
 601 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req,
 602                 struct nvme_tcp_r2t_pdu *pdu)
 603 {
 604         struct nvme_tcp_data_pdu *data = req->pdu;
 605         struct nvme_tcp_queue *queue = req->queue;
 606         struct request *rq = blk_mq_rq_from_pdu(req);
 607         u8 hdgst = nvme_tcp_hdgst_len(queue);
 608         u8 ddgst = nvme_tcp_ddgst_len(queue);
 609
 610         req->pdu_len = le32_to_cpu(pdu->r2t_length);
 611         req->pdu_sent = 0;
 612
 613         if (unlikely(!req->pdu_len)) {
 614                 dev_err(queue->ctrl->ctrl.device,
 615                         "req %d r2t len is %u, probably a bug...\n",
 616                         rq->tag, req->pdu_len);
 617                 return -EPROTO;
 618         }
 619
 620         if (unlikely(req->data_sent + req->pdu_len > req->data_len)) {
 621                 dev_err(queue->ctrl->ctrl.device,
 622                         "req %d r2t len %u exceeded data len %u (%zu sent)\n",
 623                         rq->tag, req->pdu_len, req->data_len,
 624                         req->data_sent);
 625                 return -EPROTO;
 626         }
 627
 628         if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {
 629                 dev_err(queue->ctrl->ctrl.device,
 630                         "req %d unexpected r2t offset %u (expected %zu)\n",
 631                         rq->tag, le32_to_cpu(pdu->r2t_offset),
 632                         req->data_sent);
 633                 return -EPROTO;
 634         }
 635
 636         memset(data, 0, sizeof(*data));
 637         data->hdr.type = nvme_tcp_h2c_data;
 638         data->hdr.flags = NVME_TCP_F_DATA_LAST;
 639         if (queue->hdr_digest)
 640                 data->hdr.flags |= NVME_TCP_F_HDGST;
 641         if (queue->data_digest)
 642                 data->hdr.flags |= NVME_TCP_F_DDGST;
 643         data->hdr.hlen = sizeof(*data);
 644         data->hdr.pdo = data->hdr.hlen + hdgst;
 645         data->hdr.plen =
 646                 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
 647         data->ttag = pdu->ttag;
 648         data->command_id = nvme_cid(rq);
 649         data->data_offset = pdu->r2t_offset;
 650         data->data_length = cpu_to_le32(req->pdu_len);
 651         return 0;
 652 }
 653
 654 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 655                 struct nvme_tcp_r2t_pdu *pdu)
 656 {
 657         struct nvme_tcp_request *req;
 658         struct request *rq;
 659         int ret;
 660
 661         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 662         if (!rq) {
 663                 dev_err(queue->ctrl->ctrl.device,
 664                         "got bad r2t.command_id %#x on queue %d\n",
 665                         pdu->command_id, nvme_tcp_queue_id(queue));
 666                 return -ENOENT;
 667         }
 668         req = blk_mq_rq_to_pdu(rq);
 669
 670         ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
 671         if (unlikely(ret))
 672                 return ret;
 673
 674         req->state = NVME_TCP_SEND_H2C_PDU;
 675         req->offset = 0;
 676
 677         nvme_tcp_queue_request(req, false, true);
 678
 679         return 0;
 680 }
 681
 682 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 683                 unsigned int *offset, size_t *len)
 684 {
 685         struct nvme_tcp_hdr *hdr;
 686         char *pdu = queue->pdu;
 687         size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
 688         int ret;
 689
 690         ret = skb_copy_bits(skb, *offset,
 691                 &pdu[queue->pdu_offset], rcv_len);
 692         if (unlikely(ret))
 693                 return ret;
 694
 695         queue->pdu_remaining -= rcv_len;
 696         queue->pdu_offset += rcv_len;
 697         *offset += rcv_len;
 698         *len -= rcv_len;
 699         if (queue->pdu_remaining)
 700                 return 0;
 701
 702         hdr = queue->pdu;
 703         if (queue->hdr_digest) {
 704                 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
 705                 if (unlikely(ret))
 706                         return ret;
 707         }
 708
 709
 710         if (queue->data_digest) {
 711                 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
 712                 if (unlikely(ret))
 713                         return ret;
 714         }
 715
 716         switch (hdr->type) {
 717         case nvme_tcp_c2h_data:
 718                 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
 719         case nvme_tcp_rsp:
 720                 nvme_tcp_init_recv_ctx(queue);
 721                 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
 722         case nvme_tcp_r2t:
 723                 nvme_tcp_init_recv_ctx(queue);
 724                 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
 725         default:
 726                 dev_err(queue->ctrl->ctrl.device,
 727                         "unsupported pdu type (%d)\n", hdr->type);
 728                 return -EINVAL;
 729         }
 730 }
 731
 732 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 733 {
 734         union nvme_result res = {};
 735
 736         if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
 737                 nvme_complete_rq(rq);
 738 }
 739
 740 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 741                               unsigned int *offset, size_t *len)
 742 {
 743         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 744         struct request *rq =
 745                 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 746         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 747
 748         while (true) {
 749                 int recv_len, ret;
 750
 751                 recv_len = min_t(size_t, *len, queue->data_remaining);
 752                 if (!recv_len)
 753                         break;
 754
 755                 if (!iov_iter_count(&req->iter)) {
 756                         req->curr_bio = req->curr_bio->bi_next;
 757
 758                         /*
 759                          * If we don`t have any bios it means that controller
 760                          * sent more data than we requested, hence error
 761                          */
 762                         if (!req->curr_bio) {
 763                                 dev_err(queue->ctrl->ctrl.device,
 764                                         "queue %d no space in request %#x",
 765                                         nvme_tcp_queue_id(queue), rq->tag);
 766                                 nvme_tcp_init_recv_ctx(queue);
 767                                 return -EIO;
 768                         }
 769                         nvme_tcp_init_iter(req, READ);
 770                 }
 771
 772                 /* we can read only from what is left in this bio */
 773                 recv_len = min_t(size_t, recv_len,
 774                                 iov_iter_count(&req->iter));
 775
 776                 if (queue->data_digest)
 777                         ret = skb_copy_and_hash_datagram_iter(skb, *offset,
 778                                 &req->iter, recv_len, queue->rcv_hash);
 779                 else
 780                         ret = skb_copy_datagram_iter(skb, *offset,
 781                                         &req->iter, recv_len);
 782                 if (ret) {
 783                         dev_err(queue->ctrl->ctrl.device,
 784                                 "queue %d failed to copy request %#x data",
 785                                 nvme_tcp_queue_id(queue), rq->tag);
 786                         return ret;
 787                 }
 788
 789                 *len -= recv_len;
 790                 *offset += recv_len;
 791                 queue->data_remaining -= recv_len;
 792         }
 793
 794         if (!queue->data_remaining) {
 795                 if (queue->data_digest) {
 796                         nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
 797                         queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
 798                 } else {
 799                         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 800                                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
 801                                 queue->nr_cqe++;
 802                         }
 803                         nvme_tcp_init_recv_ctx(queue);
 804                 }
 805         }
 806
 807         return 0;
 808 }
 809
 810 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
 811                 struct sk_buff *skb, unsigned int *offset, size_t *len)
 812 {
 813         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 814         char *ddgst = (char *)&queue->recv_ddgst;
 815         size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
 816         off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
 817         int ret;
 818
 819         ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
 820         if (unlikely(ret))
 821                 return ret;
 822
 823         queue->ddgst_remaining -= recv_len;
 824         *offset += recv_len;
 825         *len -= recv_len;
 826         if (queue->ddgst_remaining)
 827                 return 0;
 828
 829         if (queue->recv_ddgst != queue->exp_ddgst) {
 830                 dev_err(queue->ctrl->ctrl.device,
 831                         "data digest error: recv %#x expected %#x\n",
 832                         le32_to_cpu(queue->recv_ddgst),
 833                         le32_to_cpu(queue->exp_ddgst));
 834                 return -EIO;
 835         }
 836
 837         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 838                 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 839                                         pdu->command_id);
 840
 841                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
 842                 queue->nr_cqe++;
 843         }
 844
 845         nvme_tcp_init_recv_ctx(queue);
 846         return 0;
 847 }
 848
 849 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 850                              unsigned int offset, size_t len)
 851 {
 852         struct nvme_tcp_queue *queue = desc->arg.data;
 853         size_t consumed = len;
 854         int result;
 855
 856         while (len) {
 857                 switch (nvme_tcp_recv_state(queue)) {
 858                 case NVME_TCP_RECV_PDU:
 859                         result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
 860                         break;
 861                 case NVME_TCP_RECV_DATA:
 862                         result = nvme_tcp_recv_data(queue, skb, &offset, &len);
 863                         break;
 864                 case NVME_TCP_RECV_DDGST:
 865                         result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
 866                         break;
 867                 default:
 868                         result = -EFAULT;
 869                 }
 870                 if (result) {
 871                         dev_err(queue->ctrl->ctrl.device,
 872                                 "receive failed:  %d\n", result);
 873                         queue->rd_enabled = false;
 874                         nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 875                         return result;
 876                 }
 877         }
 878
 879         return consumed;
 880 }
 881
 882 static void nvme_tcp_data_ready(struct sock *sk)
 883 {
 884         struct nvme_tcp_queue *queue;
 885
 886         read_lock_bh(&sk->sk_callback_lock);
 887         queue = sk->sk_user_data;
 888         if (likely(queue && queue->rd_enabled) &&
 889             !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
 890                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 891         read_unlock_bh(&sk->sk_callback_lock);
 892 }
 893
 894 static void nvme_tcp_write_space(struct sock *sk)
 895 {
 896         struct nvme_tcp_queue *queue;
 897
 898         read_lock_bh(&sk->sk_callback_lock);
 899         queue = sk->sk_user_data;
 900         if (likely(queue && sk_stream_is_writeable(sk))) {
 901                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 902                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 903         }
 904         read_unlock_bh(&sk->sk_callback_lock);
 905 }
 906
 907 static void nvme_tcp_state_change(struct sock *sk)
 908 {
 909         struct nvme_tcp_queue *queue;
 910
 911         read_lock_bh(&sk->sk_callback_lock);
 912         queue = sk->sk_user_data;
 913         if (!queue)
 914                 goto done;
 915
 916         switch (sk->sk_state) {
 917         case TCP_CLOSE:
 918         case TCP_CLOSE_WAIT:
 919         case TCP_LAST_ACK:
 920         case TCP_FIN_WAIT1:
 921         case TCP_FIN_WAIT2:
 922                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 923                 break;
 924         default:
 925                 dev_info(queue->ctrl->ctrl.device,
 926                         "queue %d socket state %d\n",
 927                         nvme_tcp_queue_id(queue), sk->sk_state);
 928         }
 929
 930         queue->state_change(sk);
 931 done:
 932         read_unlock_bh(&sk->sk_callback_lock);
 933 }
 934
 935 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
 936 {
 937         queue->request = NULL;
 938 }
 939
 940 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
 941 {
 942         if (nvme_tcp_async_req(req)) {
 943                 union nvme_result res = {};
 944
 945                 nvme_complete_async_event(&req->queue->ctrl->ctrl,
 946                                 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
 947         } else {
 948                 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
 949                                 NVME_SC_HOST_PATH_ERROR);
 950         }
 951 }
 952
 953 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
 954 {
 955         struct nvme_tcp_queue *queue = req->queue;
 956         int req_data_len = req->data_len;
 957
 958         while (true) {
 959                 struct page *page = nvme_tcp_req_cur_page(req);
 960                 size_t offset = nvme_tcp_req_cur_offset(req);
 961                 size_t len = nvme_tcp_req_cur_length(req);
 962                 bool last = nvme_tcp_pdu_last_send(req, len);
 963                 int req_data_sent = req->data_sent;
 964                 int ret, flags = MSG_DONTWAIT;
 965
 966                 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
 967                         flags |= MSG_EOR;
 968                 else
 969                         flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
 970
 971                 if (sendpage_ok(page)) {
 972                         ret = kernel_sendpage(queue->sock, page, offset, len,
 973                                         flags);
 974                 } else {
 975                         ret = sock_no_sendpage(queue->sock, page, offset, len,
 976                                         flags);
 977                 }
 978                 if (ret <= 0)
 979                         return ret;
 980
 981                 if (queue->data_digest)
 982                         nvme_tcp_ddgst_update(queue->snd_hash, page,
 983                                         offset, ret);
 984
 985                 /*
 986                  * update the request iterator except for the last payload send
 987                  * in the request where we don't want to modify it as we may
 988                  * compete with the RX path completing the request.
 989                  */
 990                 if (req_data_sent + ret < req_data_len)
 991                         nvme_tcp_advance_req(req, ret);
 992
 993                 /* fully successful last send in current PDU */
 994                 if (last && ret == len) {
 995                         if (queue->data_digest) {
 996                                 nvme_tcp_ddgst_final(queue->snd_hash,
 997                                         &req->ddgst);
 998                                 req->state = NVME_TCP_SEND_DDGST;
 999                                 req->offset = 0;
1000                         } else {
1001                                 nvme_tcp_done_send_req(queue);
1002                         }
1003                         return 1;
1004                 }
1005         }
1006         return -EAGAIN;
1007 }
1008
1009 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1010 {
1011         struct nvme_tcp_queue *queue = req->queue;
1012         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1013         bool inline_data = nvme_tcp_has_inline_data(req);
1014         u8 hdgst = nvme_tcp_hdgst_len(queue);
1015         int len = sizeof(*pdu) + hdgst - req->offset;
1016         int flags = MSG_DONTWAIT;
1017         int ret;
1018
1019         if (inline_data || nvme_tcp_queue_more(queue))
1020                 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
1021         else
1022                 flags |= MSG_EOR;
1023
1024         if (queue->hdr_digest && !req->offset)
1025                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1026
1027         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1028                         offset_in_page(pdu) + req->offset, len,  flags);
1029         if (unlikely(ret <= 0))
1030                 return ret;
1031
1032         len -= ret;
1033         if (!len) {
1034                 if (inline_data) {
1035                         req->state = NVME_TCP_SEND_DATA;
1036                         if (queue->data_digest)
1037                                 crypto_ahash_init(queue->snd_hash);
1038                         nvme_tcp_init_iter(req, WRITE);
1039                 } else {
1040                         nvme_tcp_done_send_req(queue);
1041                 }
1042                 return 1;
1043         }
1044         req->offset += ret;
1045
1046         return -EAGAIN;
1047 }
1048
1049 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1050 {
1051         struct nvme_tcp_queue *queue = req->queue;
1052         struct nvme_tcp_data_pdu *pdu = req->pdu;
1053         u8 hdgst = nvme_tcp_hdgst_len(queue);
1054         int len = sizeof(*pdu) - req->offset + hdgst;
1055         int ret;
1056
1057         if (queue->hdr_digest && !req->offset)
1058                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1059
1060         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1061                         offset_in_page(pdu) + req->offset, len,
1062                         MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1063         if (unlikely(ret <= 0))
1064                 return ret;
1065
1066         len -= ret;
1067         if (!len) {
1068                 req->state = NVME_TCP_SEND_DATA;
1069                 if (queue->data_digest)
1070                         crypto_ahash_init(queue->snd_hash);
1071                 if (!req->data_sent)
1072                         nvme_tcp_init_iter(req, WRITE);
1073                 return 1;
1074         }
1075         req->offset += ret;
1076
1077         return -EAGAIN;
1078 }
1079
1080 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1081 {
1082         struct nvme_tcp_queue *queue = req->queue;
1083         size_t offset = req->offset;
1084         int ret;
1085         struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1086         struct kvec iov = {
1087                 .iov_base = (u8 *)&req->ddgst + req->offset,
1088                 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1089         };
1090
1091         if (nvme_tcp_queue_more(queue))
1092                 msg.msg_flags |= MSG_MORE;
1093         else
1094                 msg.msg_flags |= MSG_EOR;
1095
1096         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1097         if (unlikely(ret <= 0))
1098                 return ret;
1099
1100         if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1101                 nvme_tcp_done_send_req(queue);
1102                 return 1;
1103         }
1104
1105         req->offset += ret;
1106         return -EAGAIN;
1107 }
1108
1109 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1110 {
1111         struct nvme_tcp_request *req;
1112         int ret = 1;
1113
1114         if (!queue->request) {
1115                 queue->request = nvme_tcp_fetch_request(queue);
1116                 if (!queue->request)
1117                         return 0;
1118         }
1119         req = queue->request;
1120
1121         if (req->state == NVME_TCP_SEND_CMD_PDU) {
1122                 ret = nvme_tcp_try_send_cmd_pdu(req);
1123                 if (ret <= 0)
1124                         goto done;
1125                 if (!nvme_tcp_has_inline_data(req))
1126                         return ret;
1127         }
1128
1129         if (req->state == NVME_TCP_SEND_H2C_PDU) {
1130                 ret = nvme_tcp_try_send_data_pdu(req);
1131                 if (ret <= 0)
1132                         goto done;
1133         }
1134
1135         if (req->state == NVME_TCP_SEND_DATA) {
1136                 ret = nvme_tcp_try_send_data(req);
1137                 if (ret <= 0)
1138                         goto done;
1139         }
1140
1141         if (req->state == NVME_TCP_SEND_DDGST)
1142                 ret = nvme_tcp_try_send_ddgst(req);
1143 done:
1144         if (ret == -EAGAIN) {
1145                 ret = 0;
1146         } else if (ret < 0) {
1147                 dev_err(queue->ctrl->ctrl.device,
1148                         "failed to send request %d\n", ret);
1149                 nvme_tcp_fail_request(queue->request);
1150                 nvme_tcp_done_send_req(queue);
1151         }
1152         return ret;
1153 }
1154
1155 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1156 {
1157         struct socket *sock = queue->sock;
1158         struct sock *sk = sock->sk;
1159         read_descriptor_t rd_desc;
1160         int consumed;
1161
1162         rd_desc.arg.data = queue;
1163         rd_desc.count = 1;
1164         lock_sock(sk);
1165         queue->nr_cqe = 0;
1166         consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1167         release_sock(sk);
1168         return consumed;
1169 }
1170
1171 static void nvme_tcp_io_work(struct work_struct *w)
1172 {
1173         struct nvme_tcp_queue *queue =
1174                 container_of(w, struct nvme_tcp_queue, io_work);
1175         unsigned long deadline = jiffies + msecs_to_jiffies(1);
1176
1177         do {
1178                 bool pending = false;
1179                 int result;
1180
1181                 if (mutex_trylock(&queue->send_mutex)) {
1182                         result = nvme_tcp_try_send(queue);
1183                         mutex_unlock(&queue->send_mutex);
1184                         if (result > 0)
1185                                 pending = true;
1186                         else if (unlikely(result < 0))
1187                                 break;
1188                 }
1189
1190                 result = nvme_tcp_try_recv(queue);
1191                 if (result > 0)
1192                         pending = true;
1193                 else if (unlikely(result < 0))
1194                         return;
1195
1196                 if (!pending || !queue->rd_enabled)
1197                         return;
1198
1199         } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1200
1201         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1202 }
1203
1204 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1205 {
1206         struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1207
1208         ahash_request_free(queue->rcv_hash);
1209         ahash_request_free(queue->snd_hash);
1210         crypto_free_ahash(tfm);
1211 }
1212
1213 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1214 {
1215         struct crypto_ahash *tfm;
1216
1217         tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1218         if (IS_ERR(tfm))
1219                 return PTR_ERR(tfm);
1220
1221         queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1222         if (!queue->snd_hash)
1223                 goto free_tfm;
1224         ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1225
1226         queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1227         if (!queue->rcv_hash)
1228                 goto free_snd_hash;
1229         ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1230
1231         return 0;
1232 free_snd_hash:
1233         ahash_request_free(queue->snd_hash);
1234 free_tfm:
1235         crypto_free_ahash(tfm);
1236         return -ENOMEM;
1237 }
1238
1239 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1240 {
1241         struct nvme_tcp_request *async = &ctrl->async_req;
1242
1243         page_frag_free(async->pdu);
1244 }
1245
1246 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1247 {
1248         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1249         struct nvme_tcp_request *async = &ctrl->async_req;
1250         u8 hdgst = nvme_tcp_hdgst_len(queue);
1251
1252         async->pdu = page_frag_alloc(&queue->pf_cache,
1253                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1254                 GFP_KERNEL | __GFP_ZERO);
1255         if (!async->pdu)
1256                 return -ENOMEM;
1257
1258         async->queue = &ctrl->queues[0];
1259         return 0;
1260 }
1261
1262 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1263 {
1264         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1265         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1266
1267         if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1268                 return;
1269
1270         if (queue->hdr_digest || queue->data_digest)
1271                 nvme_tcp_free_crypto(queue);
1272
1273         sock_release(queue->sock);
1274         kfree(queue->pdu);
1275         mutex_destroy(&queue->queue_lock);
1276 }
1277
1278 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1279 {
1280         struct nvme_tcp_icreq_pdu *icreq;
1281         struct nvme_tcp_icresp_pdu *icresp;
1282         struct msghdr msg = {};
1283         struct kvec iov;
1284         bool ctrl_hdgst, ctrl_ddgst;
1285         int ret;
1286
1287         icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1288         if (!icreq)
1289                 return -ENOMEM;
1290
1291         icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1292         if (!icresp) {
1293                 ret = -ENOMEM;
1294                 goto free_icreq;
1295         }
1296
1297         icreq->hdr.type = nvme_tcp_icreq;
1298         icreq->hdr.hlen = sizeof(*icreq);
1299         icreq->hdr.pdo = 0;
1300         icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1301         icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1302         icreq->maxr2t = 0; /* single inflight r2t supported */
1303         icreq->hpda = 0; /* no alignment constraint */
1304         if (queue->hdr_digest)
1305                 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1306         if (queue->data_digest)
1307                 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1308
1309         iov.iov_base = icreq;
1310         iov.iov_len = sizeof(*icreq);
1311         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1312         if (ret < 0)
1313                 goto free_icresp;
1314
1315         memset(&msg, 0, sizeof(msg));
1316         iov.iov_base = icresp;
1317         iov.iov_len = sizeof(*icresp);
1318         ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1319                         iov.iov_len, msg.msg_flags);
1320         if (ret < 0)
1321                 goto free_icresp;
1322
1323         ret = -EINVAL;
1324         if (icresp->hdr.type != nvme_tcp_icresp) {
1325                 pr_err("queue %d: bad type returned %d\n",
1326                         nvme_tcp_queue_id(queue), icresp->hdr.type);
1327                 goto free_icresp;
1328         }
1329
1330         if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1331                 pr_err("queue %d: bad pdu length returned %d\n",
1332                         nvme_tcp_queue_id(queue), icresp->hdr.plen);
1333                 goto free_icresp;
1334         }
1335
1336         if (icresp->pfv != NVME_TCP_PFV_1_0) {
1337                 pr_err("queue %d: bad pfv returned %d\n",
1338                         nvme_tcp_queue_id(queue), icresp->pfv);
1339                 goto free_icresp;
1340         }
1341
1342         ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1343         if ((queue->data_digest && !ctrl_ddgst) ||
1344             (!queue->data_digest && ctrl_ddgst)) {
1345                 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1346                         nvme_tcp_queue_id(queue),
1347                         queue->data_digest ? "enabled" : "disabled",
1348                         ctrl_ddgst ? "enabled" : "disabled");
1349                 goto free_icresp;
1350         }
1351
1352         ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1353         if ((queue->hdr_digest && !ctrl_hdgst) ||
1354             (!queue->hdr_digest && ctrl_hdgst)) {
1355                 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1356                         nvme_tcp_queue_id(queue),
1357                         queue->hdr_digest ? "enabled" : "disabled",
1358                         ctrl_hdgst ? "enabled" : "disabled");
1359                 goto free_icresp;
1360         }
1361
1362         if (icresp->cpda != 0) {
1363                 pr_err("queue %d: unsupported cpda returned %d\n",
1364                         nvme_tcp_queue_id(queue), icresp->cpda);
1365                 goto free_icresp;
1366         }
1367
1368         ret = 0;
1369 free_icresp:
1370         kfree(icresp);
1371 free_icreq:
1372         kfree(icreq);
1373         return ret;
1374 }
1375
1376 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1377 {
1378         return nvme_tcp_queue_id(queue) == 0;
1379 }
1380
1381 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1382 {
1383         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1384         int qid = nvme_tcp_queue_id(queue);
1385
1386         return !nvme_tcp_admin_queue(queue) &&
1387                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1388 }
1389
1390 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1391 {
1392         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1393         int qid = nvme_tcp_queue_id(queue);
1394
1395         return !nvme_tcp_admin_queue(queue) &&
1396                 !nvme_tcp_default_queue(queue) &&
1397                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1398                           ctrl->io_queues[HCTX_TYPE_READ];
1399 }
1400
1401 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1402 {
1403         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1404         int qid = nvme_tcp_queue_id(queue);
1405
1406         return !nvme_tcp_admin_queue(queue) &&
1407                 !nvme_tcp_default_queue(queue) &&
1408                 !nvme_tcp_read_queue(queue) &&
1409                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1410                           ctrl->io_queues[HCTX_TYPE_READ] +
1411                           ctrl->io_queues[HCTX_TYPE_POLL];
1412 }
1413
1414 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1415 {
1416         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1417         int qid = nvme_tcp_queue_id(queue);
1418         int n = 0;
1419
1420         if (nvme_tcp_default_queue(queue))
1421                 n = qid - 1;
1422         else if (nvme_tcp_read_queue(queue))
1423                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1424         else if (nvme_tcp_poll_queue(queue))
1425                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1426                                 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1427         queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1428 }
1429
1430 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1431                 int qid, size_t queue_size)
1432 {
1433         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1434         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1435         int ret, rcv_pdu_size;
1436
1437         mutex_init(&queue->queue_lock);
1438         queue->ctrl = ctrl;
1439         init_llist_head(&queue->req_list);
1440         INIT_LIST_HEAD(&queue->send_list);
1441         mutex_init(&queue->send_mutex);
1442         INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1443         queue->queue_size = queue_size;
1444
1445         if (qid > 0)
1446                 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1447         else
1448                 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1449                                                 NVME_TCP_ADMIN_CCSZ;
1450
1451         ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1452                         IPPROTO_TCP, &queue->sock);
1453         if (ret) {
1454                 dev_err(nctrl->device,
1455                         "failed to create socket: %d\n", ret);
1456                 goto err_destroy_mutex;
1457         }
1458
1459         nvme_tcp_reclassify_socket(queue->sock);
1460
1461         /* Single syn retry */
1462         tcp_sock_set_syncnt(queue->sock->sk, 1);
1463
1464         /* Set TCP no delay */
1465         tcp_sock_set_nodelay(queue->sock->sk);
1466
1467         /*
1468          * Cleanup whatever is sitting in the TCP transmit queue on socket
1469          * close. This is done to prevent stale data from being sent should
1470          * the network connection be restored before TCP times out.
1471          */
1472         sock_no_linger(queue->sock->sk);
1473
1474         if (so_priority > 0)
1475                 sock_set_priority(queue->sock->sk, so_priority);
1476
1477         /* Set socket type of service */
1478         if (nctrl->opts->tos >= 0)
1479                 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1480
1481         /* Set 10 seconds timeout for icresp recvmsg */
1482         queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1483
1484         queue->sock->sk->sk_allocation = GFP_ATOMIC;
1485         nvme_tcp_set_queue_io_cpu(queue);
1486         queue->request = NULL;
1487         queue->data_remaining = 0;
1488         queue->ddgst_remaining = 0;
1489         queue->pdu_remaining = 0;
1490         queue->pdu_offset = 0;
1491         sk_set_memalloc(queue->sock->sk);
1492
1493         if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1494                 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1495                         sizeof(ctrl->src_addr));
1496                 if (ret) {
1497                         dev_err(nctrl->device,
1498                                 "failed to bind queue %d socket %d\n",
1499                                 qid, ret);
1500                         goto err_sock;
1501                 }
1502         }
1503
1504         queue->hdr_digest = nctrl->opts->hdr_digest;
1505         queue->data_digest = nctrl->opts->data_digest;
1506         if (queue->hdr_digest || queue->data_digest) {
1507                 ret = nvme_tcp_alloc_crypto(queue);
1508                 if (ret) {
1509                         dev_err(nctrl->device,
1510                                 "failed to allocate queue %d crypto\n", qid);
1511                         goto err_sock;
1512                 }
1513         }
1514
1515         rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1516                         nvme_tcp_hdgst_len(queue);
1517         queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1518         if (!queue->pdu) {
1519                 ret = -ENOMEM;
1520                 goto err_crypto;
1521         }
1522
1523         dev_dbg(nctrl->device, "connecting queue %d\n",
1524                         nvme_tcp_queue_id(queue));
1525
1526         ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1527                 sizeof(ctrl->addr), 0);
1528         if (ret) {
1529                 dev_err(nctrl->device,
1530                         "failed to connect socket: %d\n", ret);
1531                 goto err_rcv_pdu;
1532         }
1533
1534         ret = nvme_tcp_init_connection(queue);
1535         if (ret)
1536                 goto err_init_connect;
1537
1538         queue->rd_enabled = true;
1539         set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1540         nvme_tcp_init_recv_ctx(queue);
1541
1542         write_lock_bh(&queue->sock->sk->sk_callback_lock);
1543         queue->sock->sk->sk_user_data = queue;
1544         queue->state_change = queue->sock->sk->sk_state_change;
1545         queue->data_ready = queue->sock->sk->sk_data_ready;
1546         queue->write_space = queue->sock->sk->sk_write_space;
1547         queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1548         queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1549         queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1550 #ifdef CONFIG_NET_RX_BUSY_POLL
1551         queue->sock->sk->sk_ll_usec = 1;
1552 #endif
1553         write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1554
1555         return 0;
1556
1557 err_init_connect:
1558         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1559 err_rcv_pdu:
1560         kfree(queue->pdu);
1561 err_crypto:
1562         if (queue->hdr_digest || queue->data_digest)
1563                 nvme_tcp_free_crypto(queue);
1564 err_sock:
1565         sock_release(queue->sock);
1566         queue->sock = NULL;
1567 err_destroy_mutex:
1568         mutex_destroy(&queue->queue_lock);
1569         return ret;
1570 }
1571
1572 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1573 {
1574         struct socket *sock = queue->sock;
1575
1576         write_lock_bh(&sock->sk->sk_callback_lock);
1577         sock->sk->sk_user_data  = NULL;
1578         sock->sk->sk_data_ready = queue->data_ready;
1579         sock->sk->sk_state_change = queue->state_change;
1580         sock->sk->sk_write_space  = queue->write_space;
1581         write_unlock_bh(&sock->sk->sk_callback_lock);
1582 }
1583
1584 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1585 {
1586         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1587         nvme_tcp_restore_sock_calls(queue);
1588         cancel_work_sync(&queue->io_work);
1589 }
1590
1591 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1592 {
1593         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1594         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1595
1596         mutex_lock(&queue->queue_lock);
1597         if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1598                 __nvme_tcp_stop_queue(queue);
1599         mutex_unlock(&queue->queue_lock);
1600 }
1601
1602 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1603 {
1604         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1605         int ret;
1606
1607         if (idx)
1608                 ret = nvmf_connect_io_queue(nctrl, idx, false);
1609         else
1610                 ret = nvmf_connect_admin_queue(nctrl);
1611
1612         if (!ret) {
1613                 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1614         } else {
1615                 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1616                         __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1617                 dev_err(nctrl->device,
1618                         "failed to connect queue: %d ret=%d\n", idx, ret);
1619         }
1620         return ret;
1621 }
1622
1623 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1624                 bool admin)
1625 {
1626         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1627         struct blk_mq_tag_set *set;
1628         int ret;
1629
1630         if (admin) {
1631                 set = &ctrl->admin_tag_set;
1632                 memset(set, 0, sizeof(*set));
1633                 set->ops = &nvme_tcp_admin_mq_ops;
1634                 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1635                 set->reserved_tags = 2; /* connect + keep-alive */
1636                 set->numa_node = nctrl->numa_node;
1637                 set->flags = BLK_MQ_F_BLOCKING;
1638                 set->cmd_size = sizeof(struct nvme_tcp_request);
1639                 set->driver_data = ctrl;
1640                 set->nr_hw_queues = 1;
1641                 set->timeout = ADMIN_TIMEOUT;
1642         } else {
1643                 set = &ctrl->tag_set;
1644                 memset(set, 0, sizeof(*set));
1645                 set->ops = &nvme_tcp_mq_ops;
1646                 set->queue_depth = nctrl->sqsize + 1;
1647                 set->reserved_tags = 1; /* fabric connect */
1648                 set->numa_node = nctrl->numa_node;
1649                 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1650                 set->cmd_size = sizeof(struct nvme_tcp_request);
1651                 set->driver_data = ctrl;
1652                 set->nr_hw_queues = nctrl->queue_count - 1;
1653                 set->timeout = NVME_IO_TIMEOUT;
1654                 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1655         }
1656
1657         ret = blk_mq_alloc_tag_set(set);
1658         if (ret)
1659                 return ERR_PTR(ret);
1660
1661         return set;
1662 }
1663
1664 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1665 {
1666         if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1667                 cancel_work_sync(&ctrl->async_event_work);
1668                 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1669                 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1670         }
1671
1672         nvme_tcp_free_queue(ctrl, 0);
1673 }
1674
1675 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1676 {
1677         int i;
1678
1679         for (i = 1; i < ctrl->queue_count; i++)
1680                 nvme_tcp_free_queue(ctrl, i);
1681 }
1682
1683 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1684 {
1685         int i;
1686
1687         for (i = 1; i < ctrl->queue_count; i++)
1688                 nvme_tcp_stop_queue(ctrl, i);
1689 }
1690
1691 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1692 {
1693         int i, ret = 0;
1694
1695         for (i = 1; i < ctrl->queue_count; i++) {
1696                 ret = nvme_tcp_start_queue(ctrl, i);
1697                 if (ret)
1698                         goto out_stop_queues;
1699         }
1700
1701         return 0;
1702
1703 out_stop_queues:
1704         for (i--; i >= 1; i--)
1705                 nvme_tcp_stop_queue(ctrl, i);
1706         return ret;
1707 }
1708
1709 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1710 {
1711         int ret;
1712
1713         ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1714         if (ret)
1715                 return ret;
1716
1717         ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1718         if (ret)
1719                 goto out_free_queue;
1720
1721         return 0;
1722
1723 out_free_queue:
1724         nvme_tcp_free_queue(ctrl, 0);
1725         return ret;
1726 }
1727
1728 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1729 {
1730         int i, ret;
1731
1732         for (i = 1; i < ctrl->queue_count; i++) {
1733                 ret = nvme_tcp_alloc_queue(ctrl, i,
1734                                 ctrl->sqsize + 1);
1735                 if (ret)
1736                         goto out_free_queues;
1737         }
1738
1739         return 0;
1740
1741 out_free_queues:
1742         for (i--; i >= 1; i--)
1743                 nvme_tcp_free_queue(ctrl, i);
1744
1745         return ret;
1746 }
1747
1748 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1749 {
1750         unsigned int nr_io_queues;
1751
1752         nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1753         nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1754         nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1755
1756         return nr_io_queues;
1757 }
1758
1759 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1760                 unsigned int nr_io_queues)
1761 {
1762         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1763         struct nvmf_ctrl_options *opts = nctrl->opts;
1764
1765         if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1766                 /*
1767                  * separate read/write queues
1768                  * hand out dedicated default queues only after we have
1769                  * sufficient read queues.
1770                  */
1771                 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1772                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1773                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1774                         min(opts->nr_write_queues, nr_io_queues);
1775                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1776         } else {
1777                 /*
1778                  * shared read/write queues
1779                  * either no write queues were requested, or we don't have
1780                  * sufficient queue count to have dedicated default queues.
1781                  */
1782                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1783                         min(opts->nr_io_queues, nr_io_queues);
1784                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1785         }
1786
1787         if (opts->nr_poll_queues && nr_io_queues) {
1788                 /* map dedicated poll queues only if we have queues left */
1789                 ctrl->io_queues[HCTX_TYPE_POLL] =
1790                         min(opts->nr_poll_queues, nr_io_queues);
1791         }
1792 }
1793
1794 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1795 {
1796         unsigned int nr_io_queues;
1797         int ret;
1798
1799         nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1800         ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1801         if (ret)
1802                 return ret;
1803
1804         if (nr_io_queues == 0) {
1805                 dev_err(ctrl->device,
1806                         "unable to set any I/O queues\n");
1807                 return -ENOMEM;
1808         }
1809
1810         ctrl->queue_count = nr_io_queues + 1;
1811         dev_info(ctrl->device,
1812                 "creating %d I/O queues.\n", nr_io_queues);
1813
1814         nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1815
1816         return __nvme_tcp_alloc_io_queues(ctrl);
1817 }
1818
1819 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1820 {
1821         nvme_tcp_stop_io_queues(ctrl);
1822         if (remove) {
1823                 blk_cleanup_queue(ctrl->connect_q);
1824                 blk_mq_free_tag_set(ctrl->tagset);
1825         }
1826         nvme_tcp_free_io_queues(ctrl);
1827 }
1828
1829 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1830 {
1831         int ret;
1832
1833         ret = nvme_tcp_alloc_io_queues(ctrl);
1834         if (ret)
1835                 return ret;
1836
1837         if (new) {
1838                 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1839                 if (IS_ERR(ctrl->tagset)) {
1840                         ret = PTR_ERR(ctrl->tagset);
1841                         goto out_free_io_queues;
1842                 }
1843
1844                 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
1845                 if (IS_ERR(ctrl->connect_q)) {
1846                         ret = PTR_ERR(ctrl->connect_q);
1847                         goto out_free_tag_set;
1848                 }
1849         }
1850
1851         ret = nvme_tcp_start_io_queues(ctrl);
1852         if (ret)
1853                 goto out_cleanup_connect_q;
1854
1855         if (!new) {
1856                 nvme_start_queues(ctrl);
1857                 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1858                         /*
1859                          * If we timed out waiting for freeze we are likely to
1860                          * be stuck.  Fail the controller initialization just
1861                          * to be safe.
1862                          */
1863                         ret = -ENODEV;
1864                         goto out_wait_freeze_timed_out;
1865                 }
1866                 blk_mq_update_nr_hw_queues(ctrl->tagset,
1867                         ctrl->queue_count - 1);
1868                 nvme_unfreeze(ctrl);
1869         }
1870
1871         return 0;
1872
1873 out_wait_freeze_timed_out:
1874         nvme_stop_queues(ctrl);
1875         nvme_sync_io_queues(ctrl);
1876         nvme_tcp_stop_io_queues(ctrl);
1877 out_cleanup_connect_q:
1878         nvme_cancel_tagset(ctrl);
1879         if (new)
1880                 blk_cleanup_queue(ctrl->connect_q);
1881 out_free_tag_set:
1882         if (new)
1883                 blk_mq_free_tag_set(ctrl->tagset);
1884 out_free_io_queues:
1885         nvme_tcp_free_io_queues(ctrl);
1886         return ret;
1887 }
1888
1889 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1890 {
1891         nvme_tcp_stop_queue(ctrl, 0);
1892         if (remove) {
1893                 blk_cleanup_queue(ctrl->admin_q);
1894                 blk_cleanup_queue(ctrl->fabrics_q);
1895                 blk_mq_free_tag_set(ctrl->admin_tagset);
1896         }
1897         nvme_tcp_free_admin_queue(ctrl);
1898 }
1899
1900 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1901 {
1902         int error;
1903
1904         error = nvme_tcp_alloc_admin_queue(ctrl);
1905         if (error)
1906                 return error;
1907
1908         if (new) {
1909                 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1910                 if (IS_ERR(ctrl->admin_tagset)) {
1911                         error = PTR_ERR(ctrl->admin_tagset);
1912                         goto out_free_queue;
1913                 }
1914
1915                 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1916                 if (IS_ERR(ctrl->fabrics_q)) {
1917                         error = PTR_ERR(ctrl->fabrics_q);
1918                         goto out_free_tagset;
1919                 }
1920
1921                 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1922                 if (IS_ERR(ctrl->admin_q)) {
1923                         error = PTR_ERR(ctrl->admin_q);
1924                         goto out_cleanup_fabrics_q;
1925                 }
1926         }
1927
1928         error = nvme_tcp_start_queue(ctrl, 0);
1929         if (error)
1930                 goto out_cleanup_queue;
1931
1932         error = nvme_enable_ctrl(ctrl);
1933         if (error)
1934                 goto out_stop_queue;
1935
1936         blk_mq_unquiesce_queue(ctrl->admin_q);
1937
1938         error = nvme_init_identify(ctrl);
1939         if (error)
1940                 goto out_quiesce_queue;
1941
1942         return 0;
1943
1944 out_quiesce_queue:
1945         blk_mq_quiesce_queue(ctrl->admin_q);
1946         blk_sync_queue(ctrl->admin_q);
1947 out_stop_queue:
1948         nvme_tcp_stop_queue(ctrl, 0);
1949         nvme_cancel_admin_tagset(ctrl);
1950 out_cleanup_queue:
1951         if (new)
1952                 blk_cleanup_queue(ctrl->admin_q);
1953 out_cleanup_fabrics_q:
1954         if (new)
1955                 blk_cleanup_queue(ctrl->fabrics_q);
1956 out_free_tagset:
1957         if (new)
1958                 blk_mq_free_tag_set(ctrl->admin_tagset);
1959 out_free_queue:
1960         nvme_tcp_free_admin_queue(ctrl);
1961         return error;
1962 }
1963
1964 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
1965                 bool remove)
1966 {
1967         blk_mq_quiesce_queue(ctrl->admin_q);
1968         blk_sync_queue(ctrl->admin_q);
1969         nvme_tcp_stop_queue(ctrl, 0);
1970         if (ctrl->admin_tagset) {
1971                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
1972                         nvme_cancel_request, ctrl);
1973                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
1974         }
1975         if (remove)
1976                 blk_mq_unquiesce_queue(ctrl->admin_q);
1977         nvme_tcp_destroy_admin_queue(ctrl, remove);
1978 }
1979
1980 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
1981                 bool remove)
1982 {
1983         if (ctrl->queue_count <= 1)
1984                 return;
1985         blk_mq_quiesce_queue(ctrl->admin_q);
1986         nvme_start_freeze(ctrl);
1987         nvme_stop_queues(ctrl);
1988         nvme_sync_io_queues(ctrl);
1989         nvme_tcp_stop_io_queues(ctrl);
1990         if (ctrl->tagset) {
1991                 blk_mq_tagset_busy_iter(ctrl->tagset,
1992                         nvme_cancel_request, ctrl);
1993                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
1994         }
1995         if (remove)
1996                 nvme_start_queues(ctrl);
1997         nvme_tcp_destroy_io_queues(ctrl, remove);
1998 }
1999
2000 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2001 {
2002         /* If we are resetting/deleting then do nothing */
2003         if (ctrl->state != NVME_CTRL_CONNECTING) {
2004                 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
2005                         ctrl->state == NVME_CTRL_LIVE);
2006                 return;
2007         }
2008
2009         if (nvmf_should_reconnect(ctrl)) {
2010                 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2011                         ctrl->opts->reconnect_delay);
2012                 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2013                                 ctrl->opts->reconnect_delay * HZ);
2014         } else {
2015                 dev_info(ctrl->device, "Removing controller...\n");
2016                 nvme_delete_ctrl(ctrl);
2017         }
2018 }
2019
2020 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2021 {
2022         struct nvmf_ctrl_options *opts = ctrl->opts;
2023         int ret;
2024
2025         ret = nvme_tcp_configure_admin_queue(ctrl, new);
2026         if (ret)
2027                 return ret;
2028
2029         if (ctrl->icdoff) {
2030                 dev_err(ctrl->device, "icdoff is not supported!\n");
2031                 goto destroy_admin;
2032         }
2033
2034         if (opts->queue_size > ctrl->sqsize + 1)
2035                 dev_warn(ctrl->device,
2036                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
2037                         opts->queue_size, ctrl->sqsize + 1);
2038
2039         if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2040                 dev_warn(ctrl->device,
2041                         "sqsize %u > ctrl maxcmd %u, clamping down\n",
2042                         ctrl->sqsize + 1, ctrl->maxcmd);
2043                 ctrl->sqsize = ctrl->maxcmd - 1;
2044         }
2045
2046         if (ctrl->queue_count > 1) {
2047                 ret = nvme_tcp_configure_io_queues(ctrl, new);
2048                 if (ret)
2049                         goto destroy_admin;
2050         }
2051
2052         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2053                 /*
2054                  * state change failure is ok if we started ctrl delete,
2055                  * unless we're during creation of a new controller to
2056                  * avoid races with teardown flow.
2057                  */
2058                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2059                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2060                 WARN_ON_ONCE(new);
2061                 ret = -EINVAL;
2062                 goto destroy_io;
2063         }
2064
2065         nvme_start_ctrl(ctrl);
2066         return 0;
2067
2068 destroy_io:
2069         if (ctrl->queue_count > 1) {
2070                 nvme_stop_queues(ctrl);
2071                 nvme_sync_io_queues(ctrl);
2072                 nvme_tcp_stop_io_queues(ctrl);
2073                 nvme_cancel_tagset(ctrl);
2074                 nvme_tcp_destroy_io_queues(ctrl, new);
2075         }
2076 destroy_admin:
2077         blk_mq_quiesce_queue(ctrl->admin_q);
2078         blk_sync_queue(ctrl->admin_q);
2079         nvme_tcp_stop_queue(ctrl, 0);
2080         nvme_cancel_admin_tagset(ctrl);
2081         nvme_tcp_destroy_admin_queue(ctrl, new);
2082         return ret;
2083 }
2084
2085 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2086 {
2087         struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2088                         struct nvme_tcp_ctrl, connect_work);
2089         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2090
2091         ++ctrl->nr_reconnects;
2092
2093         if (nvme_tcp_setup_ctrl(ctrl, false))
2094                 goto requeue;
2095
2096         dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2097                         ctrl->nr_reconnects);
2098
2099         ctrl->nr_reconnects = 0;
2100
2101         return;
2102
2103 requeue:
2104         dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2105                         ctrl->nr_reconnects);
2106         nvme_tcp_reconnect_or_remove(ctrl);
2107 }
2108
2109 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2110 {
2111         struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2112                                 struct nvme_tcp_ctrl, err_work);
2113         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2114
2115         nvme_stop_keep_alive(ctrl);
2116         flush_work(&ctrl->async_event_work);
2117         nvme_tcp_teardown_io_queues(ctrl, false);
2118         /* unquiesce to fail fast pending requests */
2119         nvme_start_queues(ctrl);
2120         nvme_tcp_teardown_admin_queue(ctrl, false);
2121         blk_mq_unquiesce_queue(ctrl->admin_q);
2122
2123         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2124                 /* state change failure is ok if we started ctrl delete */
2125                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2126                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2127                 return;
2128         }
2129
2130         nvme_tcp_reconnect_or_remove(ctrl);
2131 }
2132
2133 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2134 {
2135         nvme_tcp_teardown_io_queues(ctrl, shutdown);
2136         blk_mq_quiesce_queue(ctrl->admin_q);
2137         if (shutdown)
2138                 nvme_shutdown_ctrl(ctrl);
2139         else
2140                 nvme_disable_ctrl(ctrl);
2141         nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2142 }
2143
2144 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2145 {
2146         nvme_tcp_teardown_ctrl(ctrl, true);
2147 }
2148
2149 static void nvme_reset_ctrl_work(struct work_struct *work)
2150 {
2151         struct nvme_ctrl *ctrl =
2152                 container_of(work, struct nvme_ctrl, reset_work);
2153
2154         nvme_stop_ctrl(ctrl);
2155         nvme_tcp_teardown_ctrl(ctrl, false);
2156
2157         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2158                 /* state change failure is ok if we started ctrl delete */
2159                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2160                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2161                 return;
2162         }
2163
2164         if (nvme_tcp_setup_ctrl(ctrl, false))
2165                 goto out_fail;
2166
2167         return;
2168
2169 out_fail:
2170         ++ctrl->nr_reconnects;
2171         nvme_tcp_reconnect_or_remove(ctrl);
2172 }
2173
2174 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2175 {
2176         cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2177         cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2178 }
2179
2180 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2181 {
2182         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2183
2184         if (list_empty(&ctrl->list))
2185                 goto free_ctrl;
2186
2187         mutex_lock(&nvme_tcp_ctrl_mutex);
2188         list_del(&ctrl->list);
2189         mutex_unlock(&nvme_tcp_ctrl_mutex);
2190
2191         nvmf_free_options(nctrl->opts);
2192 free_ctrl:
2193         kfree(ctrl->queues);
2194         kfree(ctrl);
2195 }
2196
2197 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2198 {
2199         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2200
2201         sg->addr = 0;
2202         sg->length = 0;
2203         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2204                         NVME_SGL_FMT_TRANSPORT_A;
2205 }
2206
2207 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2208                 struct nvme_command *c, u32 data_len)
2209 {
2210         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2211
2212         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2213         sg->length = cpu_to_le32(data_len);
2214         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2215 }
2216
2217 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2218                 u32 data_len)
2219 {
2220         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2221
2222         sg->addr = 0;
2223         sg->length = cpu_to_le32(data_len);
2224         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2225                         NVME_SGL_FMT_TRANSPORT_A;
2226 }
2227
2228 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2229 {
2230         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2231         struct nvme_tcp_queue *queue = &ctrl->queues[0];
2232         struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2233         struct nvme_command *cmd = &pdu->cmd;
2234         u8 hdgst = nvme_tcp_hdgst_len(queue);
2235
2236         memset(pdu, 0, sizeof(*pdu));
2237         pdu->hdr.type = nvme_tcp_cmd;
2238         if (queue->hdr_digest)
2239                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2240         pdu->hdr.hlen = sizeof(*pdu);
2241         pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2242
2243         cmd->common.opcode = nvme_admin_async_event;
2244         cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2245         cmd->common.flags |= NVME_CMD_SGL_METABUF;
2246         nvme_tcp_set_sg_null(cmd);
2247
2248         ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2249         ctrl->async_req.offset = 0;
2250         ctrl->async_req.curr_bio = NULL;
2251         ctrl->async_req.data_len = 0;
2252
2253         nvme_tcp_queue_request(&ctrl->async_req, true, true);
2254 }
2255
2256 static void nvme_tcp_complete_timed_out(struct request *rq)
2257 {
2258         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2259         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2260
2261         nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2262         if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {
2263                 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
2264                 blk_mq_complete_request(rq);
2265         }
2266 }
2267
2268 static enum blk_eh_timer_return
2269 nvme_tcp_timeout(struct request *rq, bool reserved)
2270 {
2271         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2272         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2273         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2274
2275         dev_warn(ctrl->device,
2276                 "queue %d: timeout request %#x type %d\n",
2277                 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2278
2279         if (ctrl->state != NVME_CTRL_LIVE) {
2280                 /*
2281                  * If we are resetting, connecting or deleting we should
2282                  * complete immediately because we may block controller
2283                  * teardown or setup sequence
2284                  * - ctrl disable/shutdown fabrics requests
2285                  * - connect requests
2286                  * - initialization admin requests
2287                  * - I/O requests that entered after unquiescing and
2288                  *   the controller stopped responding
2289                  *
2290                  * All other requests should be cancelled by the error
2291                  * recovery work, so it's fine that we fail it here.
2292                  */
2293                 nvme_tcp_complete_timed_out(rq);
2294                 return BLK_EH_DONE;
2295         }
2296
2297         /*
2298          * LIVE state should trigger the normal error recovery which will
2299          * handle completing this request.
2300          */
2301         nvme_tcp_error_recovery(ctrl);
2302         return BLK_EH_RESET_TIMER;
2303 }
2304
2305 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2306                         struct request *rq)
2307 {
2308         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2309         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2310         struct nvme_command *c = &pdu->cmd;
2311
2312         c->common.flags |= NVME_CMD_SGL_METABUF;
2313
2314         if (!blk_rq_nr_phys_segments(rq))
2315                 nvme_tcp_set_sg_null(c);
2316         else if (rq_data_dir(rq) == WRITE &&
2317             req->data_len <= nvme_tcp_inline_data_size(queue))
2318                 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2319         else
2320                 nvme_tcp_set_sg_host_data(c, req->data_len);
2321
2322         return 0;
2323 }
2324
2325 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2326                 struct request *rq)
2327 {
2328         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2329         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2330         struct nvme_tcp_queue *queue = req->queue;
2331         u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2332         blk_status_t ret;
2333
2334         ret = nvme_setup_cmd(ns, rq, &pdu->cmd);
2335         if (ret)
2336                 return ret;
2337
2338         req->state = NVME_TCP_SEND_CMD_PDU;
2339         req->offset = 0;
2340         req->data_sent = 0;
2341         req->pdu_len = 0;
2342         req->pdu_sent = 0;
2343         req->data_len = blk_rq_nr_phys_segments(rq) ?
2344                                 blk_rq_payload_bytes(rq) : 0;
2345         req->curr_bio = rq->bio;
2346
2347         if (rq_data_dir(rq) == WRITE &&
2348             req->data_len <= nvme_tcp_inline_data_size(queue))
2349                 req->pdu_len = req->data_len;
2350         else if (req->curr_bio)
2351                 nvme_tcp_init_iter(req, READ);
2352
2353         pdu->hdr.type = nvme_tcp_cmd;
2354         pdu->hdr.flags = 0;
2355         if (queue->hdr_digest)
2356                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2357         if (queue->data_digest && req->pdu_len) {
2358                 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2359                 ddgst = nvme_tcp_ddgst_len(queue);
2360         }
2361         pdu->hdr.hlen = sizeof(*pdu);
2362         pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2363         pdu->hdr.plen =
2364                 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2365
2366         ret = nvme_tcp_map_data(queue, rq);
2367         if (unlikely(ret)) {
2368                 nvme_cleanup_cmd(rq);
2369                 dev_err(queue->ctrl->ctrl.device,
2370                         "Failed to map data (%d)\n", ret);
2371                 return ret;
2372         }
2373
2374         return 0;
2375 }
2376
2377 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2378 {
2379         struct nvme_tcp_queue *queue = hctx->driver_data;
2380
2381         if (!llist_empty(&queue->req_list))
2382                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2383 }
2384
2385 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2386                 const struct blk_mq_queue_data *bd)
2387 {
2388         struct nvme_ns *ns = hctx->queue->queuedata;
2389         struct nvme_tcp_queue *queue = hctx->driver_data;
2390         struct request *rq = bd->rq;
2391         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2392         bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2393         blk_status_t ret;
2394
2395         if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2396                 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
2397
2398         ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2399         if (unlikely(ret))
2400                 return ret;
2401
2402         blk_mq_start_request(rq);
2403
2404         nvme_tcp_queue_request(req, true, bd->last);
2405
2406         return BLK_STS_OK;
2407 }
2408
2409 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2410 {
2411         struct nvme_tcp_ctrl *ctrl = set->driver_data;
2412         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2413
2414         if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2415                 /* separate read/write queues */
2416                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2417                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2418                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2419                 set->map[HCTX_TYPE_READ].nr_queues =
2420                         ctrl->io_queues[HCTX_TYPE_READ];
2421                 set->map[HCTX_TYPE_READ].queue_offset =
2422                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2423         } else {
2424                 /* shared read/write queues */
2425                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2426                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2427                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2428                 set->map[HCTX_TYPE_READ].nr_queues =
2429                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2430                 set->map[HCTX_TYPE_READ].queue_offset = 0;
2431         }
2432         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2433         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2434
2435         if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2436                 /* map dedicated poll queues only if we have queues left */
2437                 set->map[HCTX_TYPE_POLL].nr_queues =
2438                                 ctrl->io_queues[HCTX_TYPE_POLL];
2439                 set->map[HCTX_TYPE_POLL].queue_offset =
2440                         ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2441                         ctrl->io_queues[HCTX_TYPE_READ];
2442                 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2443         }
2444
2445         dev_info(ctrl->ctrl.device,
2446                 "mapped %d/%d/%d default/read/poll queues.\n",
2447                 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2448                 ctrl->io_queues[HCTX_TYPE_READ],
2449                 ctrl->io_queues[HCTX_TYPE_POLL]);
2450
2451         return 0;
2452 }
2453
2454 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
2455 {
2456         struct nvme_tcp_queue *queue = hctx->driver_data;
2457         struct sock *sk = queue->sock->sk;
2458
2459         if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2460                 return 0;
2461
2462         set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2463         if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2464                 sk_busy_loop(sk, true);
2465         nvme_tcp_try_recv(queue);
2466         clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2467         return queue->nr_cqe;
2468 }
2469
2470 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2471         .queue_rq       = nvme_tcp_queue_rq,
2472         .commit_rqs     = nvme_tcp_commit_rqs,
2473         .complete       = nvme_complete_rq,
2474         .init_request   = nvme_tcp_init_request,
2475         .exit_request   = nvme_tcp_exit_request,
2476         .init_hctx      = nvme_tcp_init_hctx,
2477         .timeout        = nvme_tcp_timeout,
2478         .map_queues     = nvme_tcp_map_queues,
2479         .poll           = nvme_tcp_poll,
2480 };
2481
2482 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2483         .queue_rq       = nvme_tcp_queue_rq,
2484         .complete       = nvme_complete_rq,
2485         .init_request   = nvme_tcp_init_request,
2486         .exit_request   = nvme_tcp_exit_request,
2487         .init_hctx      = nvme_tcp_init_admin_hctx,
2488         .timeout        = nvme_tcp_timeout,
2489 };
2490
2491 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2492         .name                   = "tcp",
2493         .module                 = THIS_MODULE,
2494         .flags                  = NVME_F_FABRICS,
2495         .reg_read32             = nvmf_reg_read32,
2496         .reg_read64             = nvmf_reg_read64,
2497         .reg_write32            = nvmf_reg_write32,
2498         .free_ctrl              = nvme_tcp_free_ctrl,
2499         .submit_async_event     = nvme_tcp_submit_async_event,
2500         .delete_ctrl            = nvme_tcp_delete_ctrl,
2501         .get_address            = nvmf_get_address,
2502         .stop_ctrl              = nvme_tcp_stop_ctrl,
2503 };
2504
2505 static bool
2506 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2507 {
2508         struct nvme_tcp_ctrl *ctrl;
2509         bool found = false;
2510
2511         mutex_lock(&nvme_tcp_ctrl_mutex);
2512         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2513                 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2514                 if (found)
2515                         break;
2516         }
2517         mutex_unlock(&nvme_tcp_ctrl_mutex);
2518
2519         return found;
2520 }
2521
2522 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2523                 struct nvmf_ctrl_options *opts)
2524 {
2525         struct nvme_tcp_ctrl *ctrl;
2526         int ret;
2527
2528         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2529         if (!ctrl)
2530                 return ERR_PTR(-ENOMEM);
2531
2532         INIT_LIST_HEAD(&ctrl->list);
2533         ctrl->ctrl.opts = opts;
2534         ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2535                                 opts->nr_poll_queues + 1;
2536         ctrl->ctrl.sqsize = opts->queue_size - 1;
2537         ctrl->ctrl.kato = opts->kato;
2538
2539         INIT_DELAYED_WORK(&ctrl->connect_work,
2540                         nvme_tcp_reconnect_ctrl_work);
2541         INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2542         INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2543
2544         if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2545                 opts->trsvcid =
2546                         kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2547                 if (!opts->trsvcid) {
2548                         ret = -ENOMEM;
2549                         goto out_free_ctrl;
2550                 }
2551                 opts->mask |= NVMF_OPT_TRSVCID;
2552         }
2553
2554         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2555                         opts->traddr, opts->trsvcid, &ctrl->addr);
2556         if (ret) {
2557                 pr_err("malformed address passed: %s:%s\n",
2558                         opts->traddr, opts->trsvcid);
2559                 goto out_free_ctrl;
2560         }
2561
2562         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2563                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2564                         opts->host_traddr, NULL, &ctrl->src_addr);
2565                 if (ret) {
2566                         pr_err("malformed src address passed: %s\n",
2567                                opts->host_traddr);
2568                         goto out_free_ctrl;
2569                 }
2570         }
2571
2572         if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2573                 ret = -EALREADY;
2574                 goto out_free_ctrl;
2575         }
2576
2577         ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2578                                 GFP_KERNEL);
2579         if (!ctrl->queues) {
2580                 ret = -ENOMEM;
2581                 goto out_free_ctrl;
2582         }
2583
2584         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2585         if (ret)
2586                 goto out_kfree_queues;
2587
2588         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2589                 WARN_ON_ONCE(1);
2590                 ret = -EINTR;
2591                 goto out_uninit_ctrl;
2592         }
2593
2594         ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2595         if (ret)
2596                 goto out_uninit_ctrl;
2597
2598         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2599                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2600
2601         mutex_lock(&nvme_tcp_ctrl_mutex);
2602         list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2603         mutex_unlock(&nvme_tcp_ctrl_mutex);
2604
2605         return &ctrl->ctrl;
2606
2607 out_uninit_ctrl:
2608         nvme_uninit_ctrl(&ctrl->ctrl);
2609         nvme_put_ctrl(&ctrl->ctrl);
2610         if (ret > 0)
2611                 ret = -EIO;
2612         return ERR_PTR(ret);
2613 out_kfree_queues:
2614         kfree(ctrl->queues);
2615 out_free_ctrl:
2616         kfree(ctrl);
2617         return ERR_PTR(ret);
2618 }
2619
2620 static struct nvmf_transport_ops nvme_tcp_transport = {
2621         .name           = "tcp",
2622         .module         = THIS_MODULE,
2623         .required_opts  = NVMF_OPT_TRADDR,
2624         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2625                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2626                           NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2627                           NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2628                           NVMF_OPT_TOS,
2629         .create_ctrl    = nvme_tcp_create_ctrl,
2630 };
2631
2632 static int __init nvme_tcp_init_module(void)
2633 {
2634         nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2635                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2636         if (!nvme_tcp_wq)
2637                 return -ENOMEM;
2638
2639         nvmf_register_transport(&nvme_tcp_transport);
2640         return 0;
2641 }
2642
2643 static void __exit nvme_tcp_cleanup_module(void)
2644 {
2645         struct nvme_tcp_ctrl *ctrl;
2646
2647         nvmf_unregister_transport(&nvme_tcp_transport);
2648
2649         mutex_lock(&nvme_tcp_ctrl_mutex);
2650         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2651                 nvme_delete_ctrl(&ctrl->ctrl);
2652         mutex_unlock(&nvme_tcp_ctrl_mutex);
2653         flush_workqueue(nvme_delete_wq);
2654
2655         destroy_workqueue(nvme_tcp_wq);
2656 }
2657
2658 module_init(nvme_tcp_init_module);
2659 module_exit(nvme_tcp_cleanup_module);
2660
2661 MODULE_LICENSE("GPL v2");