1 // SPDX-License-Identifier: GPL-2.0
3 * NVMe over Fabrics TCP host.
4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/slab.h>
10 #include <linux/err.h>
11 #include <linux/nvme-tcp.h>
14 #include <linux/blk-mq.h>
15 #include <crypto/hash.h>
16 #include <net/busy_poll.h>
21 struct nvme_tcp_queue;
23 /* Define the socket priority to use for connections were it is desirable
24 * that the NIC consider performing optimized packet processing or filtering.
25 * A non-zero value being sufficient to indicate general consideration of any
26 * possible optimization. Making it a module param allows for alternative
27 * values that may be unique for some NIC implementations.
29 static int so_priority;
30 module_param(so_priority, int, 0644);
31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
33 #ifdef CONFIG_DEBUG_LOCK_ALLOC
34 /* lockdep can detect a circular dependency of the form
35 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
36 * because dependencies are tracked for both nvme-tcp and user contexts. Using
37 * a separate class prevents lockdep from conflating nvme-tcp socket use with
38 * user-space socket API use.
40 static struct lock_class_key nvme_tcp_sk_key[2];
41 static struct lock_class_key nvme_tcp_slock_key[2];
43 static void nvme_tcp_reclassify_socket(struct socket *sock)
45 struct sock *sk = sock->sk;
47 if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
50 switch (sk->sk_family) {
52 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
53 &nvme_tcp_slock_key[0],
54 "sk_lock-AF_INET-NVME",
58 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
59 &nvme_tcp_slock_key[1],
60 "sk_lock-AF_INET6-NVME",
68 static void nvme_tcp_reclassify_socket(struct socket *sock) { }
71 enum nvme_tcp_send_state {
72 NVME_TCP_SEND_CMD_PDU = 0,
73 NVME_TCP_SEND_H2C_PDU,
78 struct nvme_tcp_request {
79 struct nvme_request req;
81 struct nvme_tcp_queue *queue;
86 struct list_head entry;
87 struct llist_node lentry;
96 enum nvme_tcp_send_state state;
99 enum nvme_tcp_queue_flags {
100 NVME_TCP_Q_ALLOCATED = 0,
102 NVME_TCP_Q_POLLING = 2,
105 enum nvme_tcp_recv_state {
106 NVME_TCP_RECV_PDU = 0,
111 struct nvme_tcp_ctrl;
112 struct nvme_tcp_queue {
114 struct work_struct io_work;
117 struct mutex queue_lock;
118 struct mutex send_mutex;
119 struct llist_head req_list;
120 struct list_head send_list;
126 size_t data_remaining;
127 size_t ddgst_remaining;
131 struct nvme_tcp_request *request;
134 size_t cmnd_capsule_len;
135 struct nvme_tcp_ctrl *ctrl;
141 struct ahash_request *rcv_hash;
142 struct ahash_request *snd_hash;
146 struct page_frag_cache pf_cache;
148 void (*state_change)(struct sock *);
149 void (*data_ready)(struct sock *);
150 void (*write_space)(struct sock *);
153 struct nvme_tcp_ctrl {
154 /* read only in the hot path */
155 struct nvme_tcp_queue *queues;
156 struct blk_mq_tag_set tag_set;
158 /* other member variables */
159 struct list_head list;
160 struct blk_mq_tag_set admin_tag_set;
161 struct sockaddr_storage addr;
162 struct sockaddr_storage src_addr;
163 struct nvme_ctrl ctrl;
165 struct work_struct err_work;
166 struct delayed_work connect_work;
167 struct nvme_tcp_request async_req;
168 u32 io_queues[HCTX_MAX_TYPES];
171 static LIST_HEAD(nvme_tcp_ctrl_list);
172 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
173 static struct workqueue_struct *nvme_tcp_wq;
174 static const struct blk_mq_ops nvme_tcp_mq_ops;
175 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
176 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
178 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
180 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
183 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
185 return queue - queue->ctrl->queues;
188 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
190 u32 queue_idx = nvme_tcp_queue_id(queue);
193 return queue->ctrl->admin_tag_set.tags[queue_idx];
194 return queue->ctrl->tag_set.tags[queue_idx - 1];
197 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
199 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
202 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
204 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
207 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
209 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
212 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
214 return req == &req->queue->ctrl->async_req;
217 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
221 if (unlikely(nvme_tcp_async_req(req)))
222 return false; /* async events don't have a request */
224 rq = blk_mq_rq_from_pdu(req);
226 return rq_data_dir(rq) == WRITE && req->data_len &&
227 req->data_len <= nvme_tcp_inline_data_size(req->queue);
230 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
232 return req->iter.bvec->bv_page;
235 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
237 return req->iter.bvec->bv_offset + req->iter.iov_offset;
240 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
242 return min_t(size_t, iov_iter_single_seg_count(&req->iter),
243 req->pdu_len - req->pdu_sent);
246 static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req)
248 return req->iter.iov_offset;
251 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
253 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
254 req->pdu_len - req->pdu_sent : 0;
257 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
260 return nvme_tcp_pdu_data_left(req) <= len;
263 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
266 struct request *rq = blk_mq_rq_from_pdu(req);
272 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
273 vec = &rq->special_vec;
275 size = blk_rq_payload_bytes(rq);
278 struct bio *bio = req->curr_bio;
280 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
281 nsegs = bio_segments(bio);
282 size = bio->bi_iter.bi_size;
283 offset = bio->bi_iter.bi_bvec_done;
286 iov_iter_bvec(&req->iter, dir, vec, nsegs, size);
287 req->iter.iov_offset = offset;
290 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
293 req->data_sent += len;
294 req->pdu_sent += len;
295 iov_iter_advance(&req->iter, len);
296 if (!iov_iter_count(&req->iter) &&
297 req->data_sent < req->data_len) {
298 req->curr_bio = req->curr_bio->bi_next;
299 nvme_tcp_init_iter(req, WRITE);
303 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
307 /* drain the send queue as much as we can... */
309 ret = nvme_tcp_try_send(queue);
313 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
315 return !list_empty(&queue->send_list) ||
316 !llist_empty(&queue->req_list);
319 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
320 bool sync, bool last)
322 struct nvme_tcp_queue *queue = req->queue;
325 empty = llist_add(&req->lentry, &queue->req_list) &&
326 list_empty(&queue->send_list) && !queue->request;
329 * if we're the first on the send_list and we can try to send
330 * directly, otherwise queue io_work. Also, only do that if we
331 * are on the same cpu, so we don't introduce contention.
333 if (queue->io_cpu == raw_smp_processor_id() &&
334 sync && empty && mutex_trylock(&queue->send_mutex)) {
335 nvme_tcp_send_all(queue);
336 mutex_unlock(&queue->send_mutex);
339 if (last && nvme_tcp_queue_more(queue))
340 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
343 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
345 struct nvme_tcp_request *req;
346 struct llist_node *node;
348 for (node = llist_del_all(&queue->req_list); node; node = node->next) {
349 req = llist_entry(node, struct nvme_tcp_request, lentry);
350 list_add(&req->entry, &queue->send_list);
354 static inline struct nvme_tcp_request *
355 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
357 struct nvme_tcp_request *req;
359 req = list_first_entry_or_null(&queue->send_list,
360 struct nvme_tcp_request, entry);
362 nvme_tcp_process_req_list(queue);
363 req = list_first_entry_or_null(&queue->send_list,
364 struct nvme_tcp_request, entry);
369 list_del(&req->entry);
373 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
376 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
377 crypto_ahash_final(hash);
380 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
381 struct page *page, off_t off, size_t len)
383 struct scatterlist sg;
385 sg_init_marker(&sg, 1);
386 sg_set_page(&sg, page, len, off);
387 ahash_request_set_crypt(hash, &sg, NULL, len);
388 crypto_ahash_update(hash);
391 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
392 void *pdu, size_t len)
394 struct scatterlist sg;
396 sg_init_one(&sg, pdu, len);
397 ahash_request_set_crypt(hash, &sg, pdu + len, len);
398 crypto_ahash_digest(hash);
401 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
402 void *pdu, size_t pdu_len)
404 struct nvme_tcp_hdr *hdr = pdu;
408 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
409 dev_err(queue->ctrl->ctrl.device,
410 "queue %d: header digest flag is cleared\n",
411 nvme_tcp_queue_id(queue));
415 recv_digest = *(__le32 *)(pdu + hdr->hlen);
416 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
417 exp_digest = *(__le32 *)(pdu + hdr->hlen);
418 if (recv_digest != exp_digest) {
419 dev_err(queue->ctrl->ctrl.device,
420 "header digest error: recv %#x expected %#x\n",
421 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
428 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
430 struct nvme_tcp_hdr *hdr = pdu;
431 u8 digest_len = nvme_tcp_hdgst_len(queue);
434 len = le32_to_cpu(hdr->plen) - hdr->hlen -
435 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
437 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
438 dev_err(queue->ctrl->ctrl.device,
439 "queue %d: data digest flag is cleared\n",
440 nvme_tcp_queue_id(queue));
443 crypto_ahash_init(queue->rcv_hash);
448 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
449 struct request *rq, unsigned int hctx_idx)
451 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
453 page_frag_free(req->pdu);
456 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
457 struct request *rq, unsigned int hctx_idx,
458 unsigned int numa_node)
460 struct nvme_tcp_ctrl *ctrl = set->driver_data;
461 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
462 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
463 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
464 u8 hdgst = nvme_tcp_hdgst_len(queue);
466 req->pdu = page_frag_alloc(&queue->pf_cache,
467 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
468 GFP_KERNEL | __GFP_ZERO);
473 nvme_req(rq)->ctrl = &ctrl->ctrl;
478 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
479 unsigned int hctx_idx)
481 struct nvme_tcp_ctrl *ctrl = data;
482 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
484 hctx->driver_data = queue;
488 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
489 unsigned int hctx_idx)
491 struct nvme_tcp_ctrl *ctrl = data;
492 struct nvme_tcp_queue *queue = &ctrl->queues[0];
494 hctx->driver_data = queue;
498 static enum nvme_tcp_recv_state
499 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
501 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
502 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
506 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
508 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
509 nvme_tcp_hdgst_len(queue);
510 queue->pdu_offset = 0;
511 queue->data_remaining = -1;
512 queue->ddgst_remaining = 0;
515 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
517 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
520 dev_warn(ctrl->device, "starting error recovery\n");
521 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
524 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
525 struct nvme_completion *cqe)
529 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
531 dev_err(queue->ctrl->ctrl.device,
532 "got bad cqe.command_id %#x on queue %d\n",
533 cqe->command_id, nvme_tcp_queue_id(queue));
534 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
538 if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
539 nvme_complete_rq(rq);
545 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
546 struct nvme_tcp_data_pdu *pdu)
550 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
552 dev_err(queue->ctrl->ctrl.device,
553 "got bad c2hdata.command_id %#x on queue %d\n",
554 pdu->command_id, nvme_tcp_queue_id(queue));
558 if (!blk_rq_payload_bytes(rq)) {
559 dev_err(queue->ctrl->ctrl.device,
560 "queue %d tag %#x unexpected data\n",
561 nvme_tcp_queue_id(queue), rq->tag);
565 queue->data_remaining = le32_to_cpu(pdu->data_length);
567 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
568 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
569 dev_err(queue->ctrl->ctrl.device,
570 "queue %d tag %#x SUCCESS set but not last PDU\n",
571 nvme_tcp_queue_id(queue), rq->tag);
572 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
579 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
580 struct nvme_tcp_rsp_pdu *pdu)
582 struct nvme_completion *cqe = &pdu->cqe;
586 * AEN requests are special as they don't time out and can
587 * survive any kind of queue freeze and often don't respond to
588 * aborts. We don't even bother to allocate a struct request
589 * for them but rather special case them here.
591 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
593 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
596 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
601 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req,
602 struct nvme_tcp_r2t_pdu *pdu)
604 struct nvme_tcp_data_pdu *data = req->pdu;
605 struct nvme_tcp_queue *queue = req->queue;
606 struct request *rq = blk_mq_rq_from_pdu(req);
607 u8 hdgst = nvme_tcp_hdgst_len(queue);
608 u8 ddgst = nvme_tcp_ddgst_len(queue);
610 req->pdu_len = le32_to_cpu(pdu->r2t_length);
613 if (unlikely(!req->pdu_len)) {
614 dev_err(queue->ctrl->ctrl.device,
615 "req %d r2t len is %u, probably a bug...\n",
616 rq->tag, req->pdu_len);
620 if (unlikely(req->data_sent + req->pdu_len > req->data_len)) {
621 dev_err(queue->ctrl->ctrl.device,
622 "req %d r2t len %u exceeded data len %u (%zu sent)\n",
623 rq->tag, req->pdu_len, req->data_len,
628 if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {
629 dev_err(queue->ctrl->ctrl.device,
630 "req %d unexpected r2t offset %u (expected %zu)\n",
631 rq->tag, le32_to_cpu(pdu->r2t_offset),
636 memset(data, 0, sizeof(*data));
637 data->hdr.type = nvme_tcp_h2c_data;
638 data->hdr.flags = NVME_TCP_F_DATA_LAST;
639 if (queue->hdr_digest)
640 data->hdr.flags |= NVME_TCP_F_HDGST;
641 if (queue->data_digest)
642 data->hdr.flags |= NVME_TCP_F_DDGST;
643 data->hdr.hlen = sizeof(*data);
644 data->hdr.pdo = data->hdr.hlen + hdgst;
646 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
647 data->ttag = pdu->ttag;
648 data->command_id = nvme_cid(rq);
649 data->data_offset = pdu->r2t_offset;
650 data->data_length = cpu_to_le32(req->pdu_len);
654 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
655 struct nvme_tcp_r2t_pdu *pdu)
657 struct nvme_tcp_request *req;
661 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
663 dev_err(queue->ctrl->ctrl.device,
664 "got bad r2t.command_id %#x on queue %d\n",
665 pdu->command_id, nvme_tcp_queue_id(queue));
668 req = blk_mq_rq_to_pdu(rq);
670 ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
674 req->state = NVME_TCP_SEND_H2C_PDU;
677 nvme_tcp_queue_request(req, false, true);
682 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
683 unsigned int *offset, size_t *len)
685 struct nvme_tcp_hdr *hdr;
686 char *pdu = queue->pdu;
687 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
690 ret = skb_copy_bits(skb, *offset,
691 &pdu[queue->pdu_offset], rcv_len);
695 queue->pdu_remaining -= rcv_len;
696 queue->pdu_offset += rcv_len;
699 if (queue->pdu_remaining)
703 if (queue->hdr_digest) {
704 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
710 if (queue->data_digest) {
711 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
717 case nvme_tcp_c2h_data:
718 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
720 nvme_tcp_init_recv_ctx(queue);
721 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
723 nvme_tcp_init_recv_ctx(queue);
724 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
726 dev_err(queue->ctrl->ctrl.device,
727 "unsupported pdu type (%d)\n", hdr->type);
732 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
734 union nvme_result res = {};
736 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
737 nvme_complete_rq(rq);
740 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
741 unsigned int *offset, size_t *len)
743 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
745 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
746 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
751 recv_len = min_t(size_t, *len, queue->data_remaining);
755 if (!iov_iter_count(&req->iter)) {
756 req->curr_bio = req->curr_bio->bi_next;
759 * If we don`t have any bios it means that controller
760 * sent more data than we requested, hence error
762 if (!req->curr_bio) {
763 dev_err(queue->ctrl->ctrl.device,
764 "queue %d no space in request %#x",
765 nvme_tcp_queue_id(queue), rq->tag);
766 nvme_tcp_init_recv_ctx(queue);
769 nvme_tcp_init_iter(req, READ);
772 /* we can read only from what is left in this bio */
773 recv_len = min_t(size_t, recv_len,
774 iov_iter_count(&req->iter));
776 if (queue->data_digest)
777 ret = skb_copy_and_hash_datagram_iter(skb, *offset,
778 &req->iter, recv_len, queue->rcv_hash);
780 ret = skb_copy_datagram_iter(skb, *offset,
781 &req->iter, recv_len);
783 dev_err(queue->ctrl->ctrl.device,
784 "queue %d failed to copy request %#x data",
785 nvme_tcp_queue_id(queue), rq->tag);
791 queue->data_remaining -= recv_len;
794 if (!queue->data_remaining) {
795 if (queue->data_digest) {
796 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
797 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
799 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
800 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
803 nvme_tcp_init_recv_ctx(queue);
810 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
811 struct sk_buff *skb, unsigned int *offset, size_t *len)
813 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
814 char *ddgst = (char *)&queue->recv_ddgst;
815 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
816 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
819 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
823 queue->ddgst_remaining -= recv_len;
826 if (queue->ddgst_remaining)
829 if (queue->recv_ddgst != queue->exp_ddgst) {
830 dev_err(queue->ctrl->ctrl.device,
831 "data digest error: recv %#x expected %#x\n",
832 le32_to_cpu(queue->recv_ddgst),
833 le32_to_cpu(queue->exp_ddgst));
837 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
838 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
841 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
845 nvme_tcp_init_recv_ctx(queue);
849 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
850 unsigned int offset, size_t len)
852 struct nvme_tcp_queue *queue = desc->arg.data;
853 size_t consumed = len;
857 switch (nvme_tcp_recv_state(queue)) {
858 case NVME_TCP_RECV_PDU:
859 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
861 case NVME_TCP_RECV_DATA:
862 result = nvme_tcp_recv_data(queue, skb, &offset, &len);
864 case NVME_TCP_RECV_DDGST:
865 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
871 dev_err(queue->ctrl->ctrl.device,
872 "receive failed: %d\n", result);
873 queue->rd_enabled = false;
874 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
882 static void nvme_tcp_data_ready(struct sock *sk)
884 struct nvme_tcp_queue *queue;
886 read_lock_bh(&sk->sk_callback_lock);
887 queue = sk->sk_user_data;
888 if (likely(queue && queue->rd_enabled) &&
889 !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
890 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
891 read_unlock_bh(&sk->sk_callback_lock);
894 static void nvme_tcp_write_space(struct sock *sk)
896 struct nvme_tcp_queue *queue;
898 read_lock_bh(&sk->sk_callback_lock);
899 queue = sk->sk_user_data;
900 if (likely(queue && sk_stream_is_writeable(sk))) {
901 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
902 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
904 read_unlock_bh(&sk->sk_callback_lock);
907 static void nvme_tcp_state_change(struct sock *sk)
909 struct nvme_tcp_queue *queue;
911 read_lock_bh(&sk->sk_callback_lock);
912 queue = sk->sk_user_data;
916 switch (sk->sk_state) {
922 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
925 dev_info(queue->ctrl->ctrl.device,
926 "queue %d socket state %d\n",
927 nvme_tcp_queue_id(queue), sk->sk_state);
930 queue->state_change(sk);
932 read_unlock_bh(&sk->sk_callback_lock);
935 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
937 queue->request = NULL;
940 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
942 if (nvme_tcp_async_req(req)) {
943 union nvme_result res = {};
945 nvme_complete_async_event(&req->queue->ctrl->ctrl,
946 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
948 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
949 NVME_SC_HOST_PATH_ERROR);
953 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
955 struct nvme_tcp_queue *queue = req->queue;
956 int req_data_len = req->data_len;
959 struct page *page = nvme_tcp_req_cur_page(req);
960 size_t offset = nvme_tcp_req_cur_offset(req);
961 size_t len = nvme_tcp_req_cur_length(req);
962 bool last = nvme_tcp_pdu_last_send(req, len);
963 int req_data_sent = req->data_sent;
964 int ret, flags = MSG_DONTWAIT;
966 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
969 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
971 if (sendpage_ok(page)) {
972 ret = kernel_sendpage(queue->sock, page, offset, len,
975 ret = sock_no_sendpage(queue->sock, page, offset, len,
981 if (queue->data_digest)
982 nvme_tcp_ddgst_update(queue->snd_hash, page,
986 * update the request iterator except for the last payload send
987 * in the request where we don't want to modify it as we may
988 * compete with the RX path completing the request.
990 if (req_data_sent + ret < req_data_len)
991 nvme_tcp_advance_req(req, ret);
993 /* fully successful last send in current PDU */
994 if (last && ret == len) {
995 if (queue->data_digest) {
996 nvme_tcp_ddgst_final(queue->snd_hash,
998 req->state = NVME_TCP_SEND_DDGST;
1001 nvme_tcp_done_send_req(queue);
1009 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1011 struct nvme_tcp_queue *queue = req->queue;
1012 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1013 bool inline_data = nvme_tcp_has_inline_data(req);
1014 u8 hdgst = nvme_tcp_hdgst_len(queue);
1015 int len = sizeof(*pdu) + hdgst - req->offset;
1016 int flags = MSG_DONTWAIT;
1019 if (inline_data || nvme_tcp_queue_more(queue))
1020 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
1024 if (queue->hdr_digest && !req->offset)
1025 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1027 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1028 offset_in_page(pdu) + req->offset, len, flags);
1029 if (unlikely(ret <= 0))
1035 req->state = NVME_TCP_SEND_DATA;
1036 if (queue->data_digest)
1037 crypto_ahash_init(queue->snd_hash);
1038 nvme_tcp_init_iter(req, WRITE);
1040 nvme_tcp_done_send_req(queue);
1049 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1051 struct nvme_tcp_queue *queue = req->queue;
1052 struct nvme_tcp_data_pdu *pdu = req->pdu;
1053 u8 hdgst = nvme_tcp_hdgst_len(queue);
1054 int len = sizeof(*pdu) - req->offset + hdgst;
1057 if (queue->hdr_digest && !req->offset)
1058 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1060 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1061 offset_in_page(pdu) + req->offset, len,
1062 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1063 if (unlikely(ret <= 0))
1068 req->state = NVME_TCP_SEND_DATA;
1069 if (queue->data_digest)
1070 crypto_ahash_init(queue->snd_hash);
1071 if (!req->data_sent)
1072 nvme_tcp_init_iter(req, WRITE);
1080 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1082 struct nvme_tcp_queue *queue = req->queue;
1083 size_t offset = req->offset;
1085 struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1087 .iov_base = (u8 *)&req->ddgst + req->offset,
1088 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1091 if (nvme_tcp_queue_more(queue))
1092 msg.msg_flags |= MSG_MORE;
1094 msg.msg_flags |= MSG_EOR;
1096 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1097 if (unlikely(ret <= 0))
1100 if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1101 nvme_tcp_done_send_req(queue);
1109 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1111 struct nvme_tcp_request *req;
1114 if (!queue->request) {
1115 queue->request = nvme_tcp_fetch_request(queue);
1116 if (!queue->request)
1119 req = queue->request;
1121 if (req->state == NVME_TCP_SEND_CMD_PDU) {
1122 ret = nvme_tcp_try_send_cmd_pdu(req);
1125 if (!nvme_tcp_has_inline_data(req))
1129 if (req->state == NVME_TCP_SEND_H2C_PDU) {
1130 ret = nvme_tcp_try_send_data_pdu(req);
1135 if (req->state == NVME_TCP_SEND_DATA) {
1136 ret = nvme_tcp_try_send_data(req);
1141 if (req->state == NVME_TCP_SEND_DDGST)
1142 ret = nvme_tcp_try_send_ddgst(req);
1144 if (ret == -EAGAIN) {
1146 } else if (ret < 0) {
1147 dev_err(queue->ctrl->ctrl.device,
1148 "failed to send request %d\n", ret);
1149 nvme_tcp_fail_request(queue->request);
1150 nvme_tcp_done_send_req(queue);
1155 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1157 struct socket *sock = queue->sock;
1158 struct sock *sk = sock->sk;
1159 read_descriptor_t rd_desc;
1162 rd_desc.arg.data = queue;
1166 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1171 static void nvme_tcp_io_work(struct work_struct *w)
1173 struct nvme_tcp_queue *queue =
1174 container_of(w, struct nvme_tcp_queue, io_work);
1175 unsigned long deadline = jiffies + msecs_to_jiffies(1);
1178 bool pending = false;
1181 if (mutex_trylock(&queue->send_mutex)) {
1182 result = nvme_tcp_try_send(queue);
1183 mutex_unlock(&queue->send_mutex);
1186 else if (unlikely(result < 0))
1190 result = nvme_tcp_try_recv(queue);
1193 else if (unlikely(result < 0))
1196 if (!pending || !queue->rd_enabled)
1199 } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1201 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1204 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1206 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1208 ahash_request_free(queue->rcv_hash);
1209 ahash_request_free(queue->snd_hash);
1210 crypto_free_ahash(tfm);
1213 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1215 struct crypto_ahash *tfm;
1217 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1219 return PTR_ERR(tfm);
1221 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1222 if (!queue->snd_hash)
1224 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1226 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1227 if (!queue->rcv_hash)
1229 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1233 ahash_request_free(queue->snd_hash);
1235 crypto_free_ahash(tfm);
1239 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1241 struct nvme_tcp_request *async = &ctrl->async_req;
1243 page_frag_free(async->pdu);
1246 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1248 struct nvme_tcp_queue *queue = &ctrl->queues[0];
1249 struct nvme_tcp_request *async = &ctrl->async_req;
1250 u8 hdgst = nvme_tcp_hdgst_len(queue);
1252 async->pdu = page_frag_alloc(&queue->pf_cache,
1253 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1254 GFP_KERNEL | __GFP_ZERO);
1258 async->queue = &ctrl->queues[0];
1262 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1264 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1265 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1267 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1270 if (queue->hdr_digest || queue->data_digest)
1271 nvme_tcp_free_crypto(queue);
1273 sock_release(queue->sock);
1275 mutex_destroy(&queue->queue_lock);
1278 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1280 struct nvme_tcp_icreq_pdu *icreq;
1281 struct nvme_tcp_icresp_pdu *icresp;
1282 struct msghdr msg = {};
1284 bool ctrl_hdgst, ctrl_ddgst;
1287 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1291 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1297 icreq->hdr.type = nvme_tcp_icreq;
1298 icreq->hdr.hlen = sizeof(*icreq);
1300 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1301 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1302 icreq->maxr2t = 0; /* single inflight r2t supported */
1303 icreq->hpda = 0; /* no alignment constraint */
1304 if (queue->hdr_digest)
1305 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1306 if (queue->data_digest)
1307 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1309 iov.iov_base = icreq;
1310 iov.iov_len = sizeof(*icreq);
1311 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1315 memset(&msg, 0, sizeof(msg));
1316 iov.iov_base = icresp;
1317 iov.iov_len = sizeof(*icresp);
1318 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1319 iov.iov_len, msg.msg_flags);
1324 if (icresp->hdr.type != nvme_tcp_icresp) {
1325 pr_err("queue %d: bad type returned %d\n",
1326 nvme_tcp_queue_id(queue), icresp->hdr.type);
1330 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1331 pr_err("queue %d: bad pdu length returned %d\n",
1332 nvme_tcp_queue_id(queue), icresp->hdr.plen);
1336 if (icresp->pfv != NVME_TCP_PFV_1_0) {
1337 pr_err("queue %d: bad pfv returned %d\n",
1338 nvme_tcp_queue_id(queue), icresp->pfv);
1342 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1343 if ((queue->data_digest && !ctrl_ddgst) ||
1344 (!queue->data_digest && ctrl_ddgst)) {
1345 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1346 nvme_tcp_queue_id(queue),
1347 queue->data_digest ? "enabled" : "disabled",
1348 ctrl_ddgst ? "enabled" : "disabled");
1352 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1353 if ((queue->hdr_digest && !ctrl_hdgst) ||
1354 (!queue->hdr_digest && ctrl_hdgst)) {
1355 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1356 nvme_tcp_queue_id(queue),
1357 queue->hdr_digest ? "enabled" : "disabled",
1358 ctrl_hdgst ? "enabled" : "disabled");
1362 if (icresp->cpda != 0) {
1363 pr_err("queue %d: unsupported cpda returned %d\n",
1364 nvme_tcp_queue_id(queue), icresp->cpda);
1376 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1378 return nvme_tcp_queue_id(queue) == 0;
1381 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1383 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1384 int qid = nvme_tcp_queue_id(queue);
1386 return !nvme_tcp_admin_queue(queue) &&
1387 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1390 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1392 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1393 int qid = nvme_tcp_queue_id(queue);
1395 return !nvme_tcp_admin_queue(queue) &&
1396 !nvme_tcp_default_queue(queue) &&
1397 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1398 ctrl->io_queues[HCTX_TYPE_READ];
1401 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1403 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1404 int qid = nvme_tcp_queue_id(queue);
1406 return !nvme_tcp_admin_queue(queue) &&
1407 !nvme_tcp_default_queue(queue) &&
1408 !nvme_tcp_read_queue(queue) &&
1409 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1410 ctrl->io_queues[HCTX_TYPE_READ] +
1411 ctrl->io_queues[HCTX_TYPE_POLL];
1414 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1416 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1417 int qid = nvme_tcp_queue_id(queue);
1420 if (nvme_tcp_default_queue(queue))
1422 else if (nvme_tcp_read_queue(queue))
1423 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1424 else if (nvme_tcp_poll_queue(queue))
1425 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1426 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1427 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1430 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1431 int qid, size_t queue_size)
1433 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1434 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1435 int ret, rcv_pdu_size;
1437 mutex_init(&queue->queue_lock);
1439 init_llist_head(&queue->req_list);
1440 INIT_LIST_HEAD(&queue->send_list);
1441 mutex_init(&queue->send_mutex);
1442 INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1443 queue->queue_size = queue_size;
1446 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1448 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1449 NVME_TCP_ADMIN_CCSZ;
1451 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1452 IPPROTO_TCP, &queue->sock);
1454 dev_err(nctrl->device,
1455 "failed to create socket: %d\n", ret);
1456 goto err_destroy_mutex;
1459 nvme_tcp_reclassify_socket(queue->sock);
1461 /* Single syn retry */
1462 tcp_sock_set_syncnt(queue->sock->sk, 1);
1464 /* Set TCP no delay */
1465 tcp_sock_set_nodelay(queue->sock->sk);
1468 * Cleanup whatever is sitting in the TCP transmit queue on socket
1469 * close. This is done to prevent stale data from being sent should
1470 * the network connection be restored before TCP times out.
1472 sock_no_linger(queue->sock->sk);
1474 if (so_priority > 0)
1475 sock_set_priority(queue->sock->sk, so_priority);
1477 /* Set socket type of service */
1478 if (nctrl->opts->tos >= 0)
1479 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1481 /* Set 10 seconds timeout for icresp recvmsg */
1482 queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1484 queue->sock->sk->sk_allocation = GFP_ATOMIC;
1485 nvme_tcp_set_queue_io_cpu(queue);
1486 queue->request = NULL;
1487 queue->data_remaining = 0;
1488 queue->ddgst_remaining = 0;
1489 queue->pdu_remaining = 0;
1490 queue->pdu_offset = 0;
1491 sk_set_memalloc(queue->sock->sk);
1493 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1494 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1495 sizeof(ctrl->src_addr));
1497 dev_err(nctrl->device,
1498 "failed to bind queue %d socket %d\n",
1504 queue->hdr_digest = nctrl->opts->hdr_digest;
1505 queue->data_digest = nctrl->opts->data_digest;
1506 if (queue->hdr_digest || queue->data_digest) {
1507 ret = nvme_tcp_alloc_crypto(queue);
1509 dev_err(nctrl->device,
1510 "failed to allocate queue %d crypto\n", qid);
1515 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1516 nvme_tcp_hdgst_len(queue);
1517 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1523 dev_dbg(nctrl->device, "connecting queue %d\n",
1524 nvme_tcp_queue_id(queue));
1526 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1527 sizeof(ctrl->addr), 0);
1529 dev_err(nctrl->device,
1530 "failed to connect socket: %d\n", ret);
1534 ret = nvme_tcp_init_connection(queue);
1536 goto err_init_connect;
1538 queue->rd_enabled = true;
1539 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1540 nvme_tcp_init_recv_ctx(queue);
1542 write_lock_bh(&queue->sock->sk->sk_callback_lock);
1543 queue->sock->sk->sk_user_data = queue;
1544 queue->state_change = queue->sock->sk->sk_state_change;
1545 queue->data_ready = queue->sock->sk->sk_data_ready;
1546 queue->write_space = queue->sock->sk->sk_write_space;
1547 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1548 queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1549 queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1550 #ifdef CONFIG_NET_RX_BUSY_POLL
1551 queue->sock->sk->sk_ll_usec = 1;
1553 write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1558 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1562 if (queue->hdr_digest || queue->data_digest)
1563 nvme_tcp_free_crypto(queue);
1565 sock_release(queue->sock);
1568 mutex_destroy(&queue->queue_lock);
1572 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1574 struct socket *sock = queue->sock;
1576 write_lock_bh(&sock->sk->sk_callback_lock);
1577 sock->sk->sk_user_data = NULL;
1578 sock->sk->sk_data_ready = queue->data_ready;
1579 sock->sk->sk_state_change = queue->state_change;
1580 sock->sk->sk_write_space = queue->write_space;
1581 write_unlock_bh(&sock->sk->sk_callback_lock);
1584 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1586 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1587 nvme_tcp_restore_sock_calls(queue);
1588 cancel_work_sync(&queue->io_work);
1591 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1593 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1594 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1596 mutex_lock(&queue->queue_lock);
1597 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1598 __nvme_tcp_stop_queue(queue);
1599 mutex_unlock(&queue->queue_lock);
1602 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1604 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1608 ret = nvmf_connect_io_queue(nctrl, idx, false);
1610 ret = nvmf_connect_admin_queue(nctrl);
1613 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1615 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1616 __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1617 dev_err(nctrl->device,
1618 "failed to connect queue: %d ret=%d\n", idx, ret);
1623 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1626 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1627 struct blk_mq_tag_set *set;
1631 set = &ctrl->admin_tag_set;
1632 memset(set, 0, sizeof(*set));
1633 set->ops = &nvme_tcp_admin_mq_ops;
1634 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1635 set->reserved_tags = 2; /* connect + keep-alive */
1636 set->numa_node = nctrl->numa_node;
1637 set->flags = BLK_MQ_F_BLOCKING;
1638 set->cmd_size = sizeof(struct nvme_tcp_request);
1639 set->driver_data = ctrl;
1640 set->nr_hw_queues = 1;
1641 set->timeout = ADMIN_TIMEOUT;
1643 set = &ctrl->tag_set;
1644 memset(set, 0, sizeof(*set));
1645 set->ops = &nvme_tcp_mq_ops;
1646 set->queue_depth = nctrl->sqsize + 1;
1647 set->reserved_tags = 1; /* fabric connect */
1648 set->numa_node = nctrl->numa_node;
1649 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1650 set->cmd_size = sizeof(struct nvme_tcp_request);
1651 set->driver_data = ctrl;
1652 set->nr_hw_queues = nctrl->queue_count - 1;
1653 set->timeout = NVME_IO_TIMEOUT;
1654 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1657 ret = blk_mq_alloc_tag_set(set);
1659 return ERR_PTR(ret);
1664 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1666 if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1667 cancel_work_sync(&ctrl->async_event_work);
1668 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1669 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1672 nvme_tcp_free_queue(ctrl, 0);
1675 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1679 for (i = 1; i < ctrl->queue_count; i++)
1680 nvme_tcp_free_queue(ctrl, i);
1683 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1687 for (i = 1; i < ctrl->queue_count; i++)
1688 nvme_tcp_stop_queue(ctrl, i);
1691 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1695 for (i = 1; i < ctrl->queue_count; i++) {
1696 ret = nvme_tcp_start_queue(ctrl, i);
1698 goto out_stop_queues;
1704 for (i--; i >= 1; i--)
1705 nvme_tcp_stop_queue(ctrl, i);
1709 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1713 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1717 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1719 goto out_free_queue;
1724 nvme_tcp_free_queue(ctrl, 0);
1728 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1732 for (i = 1; i < ctrl->queue_count; i++) {
1733 ret = nvme_tcp_alloc_queue(ctrl, i,
1736 goto out_free_queues;
1742 for (i--; i >= 1; i--)
1743 nvme_tcp_free_queue(ctrl, i);
1748 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1750 unsigned int nr_io_queues;
1752 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1753 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1754 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1756 return nr_io_queues;
1759 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1760 unsigned int nr_io_queues)
1762 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1763 struct nvmf_ctrl_options *opts = nctrl->opts;
1765 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1767 * separate read/write queues
1768 * hand out dedicated default queues only after we have
1769 * sufficient read queues.
1771 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1772 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1773 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1774 min(opts->nr_write_queues, nr_io_queues);
1775 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1778 * shared read/write queues
1779 * either no write queues were requested, or we don't have
1780 * sufficient queue count to have dedicated default queues.
1782 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1783 min(opts->nr_io_queues, nr_io_queues);
1784 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1787 if (opts->nr_poll_queues && nr_io_queues) {
1788 /* map dedicated poll queues only if we have queues left */
1789 ctrl->io_queues[HCTX_TYPE_POLL] =
1790 min(opts->nr_poll_queues, nr_io_queues);
1794 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1796 unsigned int nr_io_queues;
1799 nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1800 ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1804 if (nr_io_queues == 0) {
1805 dev_err(ctrl->device,
1806 "unable to set any I/O queues\n");
1810 ctrl->queue_count = nr_io_queues + 1;
1811 dev_info(ctrl->device,
1812 "creating %d I/O queues.\n", nr_io_queues);
1814 nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1816 return __nvme_tcp_alloc_io_queues(ctrl);
1819 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1821 nvme_tcp_stop_io_queues(ctrl);
1823 blk_cleanup_queue(ctrl->connect_q);
1824 blk_mq_free_tag_set(ctrl->tagset);
1826 nvme_tcp_free_io_queues(ctrl);
1829 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1833 ret = nvme_tcp_alloc_io_queues(ctrl);
1838 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1839 if (IS_ERR(ctrl->tagset)) {
1840 ret = PTR_ERR(ctrl->tagset);
1841 goto out_free_io_queues;
1844 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
1845 if (IS_ERR(ctrl->connect_q)) {
1846 ret = PTR_ERR(ctrl->connect_q);
1847 goto out_free_tag_set;
1851 ret = nvme_tcp_start_io_queues(ctrl);
1853 goto out_cleanup_connect_q;
1856 nvme_start_queues(ctrl);
1857 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1859 * If we timed out waiting for freeze we are likely to
1860 * be stuck. Fail the controller initialization just
1864 goto out_wait_freeze_timed_out;
1866 blk_mq_update_nr_hw_queues(ctrl->tagset,
1867 ctrl->queue_count - 1);
1868 nvme_unfreeze(ctrl);
1873 out_wait_freeze_timed_out:
1874 nvme_stop_queues(ctrl);
1875 nvme_sync_io_queues(ctrl);
1876 nvme_tcp_stop_io_queues(ctrl);
1877 out_cleanup_connect_q:
1878 nvme_cancel_tagset(ctrl);
1880 blk_cleanup_queue(ctrl->connect_q);
1883 blk_mq_free_tag_set(ctrl->tagset);
1885 nvme_tcp_free_io_queues(ctrl);
1889 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1891 nvme_tcp_stop_queue(ctrl, 0);
1893 blk_cleanup_queue(ctrl->admin_q);
1894 blk_cleanup_queue(ctrl->fabrics_q);
1895 blk_mq_free_tag_set(ctrl->admin_tagset);
1897 nvme_tcp_free_admin_queue(ctrl);
1900 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1904 error = nvme_tcp_alloc_admin_queue(ctrl);
1909 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1910 if (IS_ERR(ctrl->admin_tagset)) {
1911 error = PTR_ERR(ctrl->admin_tagset);
1912 goto out_free_queue;
1915 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1916 if (IS_ERR(ctrl->fabrics_q)) {
1917 error = PTR_ERR(ctrl->fabrics_q);
1918 goto out_free_tagset;
1921 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1922 if (IS_ERR(ctrl->admin_q)) {
1923 error = PTR_ERR(ctrl->admin_q);
1924 goto out_cleanup_fabrics_q;
1928 error = nvme_tcp_start_queue(ctrl, 0);
1930 goto out_cleanup_queue;
1932 error = nvme_enable_ctrl(ctrl);
1934 goto out_stop_queue;
1936 blk_mq_unquiesce_queue(ctrl->admin_q);
1938 error = nvme_init_identify(ctrl);
1940 goto out_quiesce_queue;
1945 blk_mq_quiesce_queue(ctrl->admin_q);
1946 blk_sync_queue(ctrl->admin_q);
1948 nvme_tcp_stop_queue(ctrl, 0);
1949 nvme_cancel_admin_tagset(ctrl);
1952 blk_cleanup_queue(ctrl->admin_q);
1953 out_cleanup_fabrics_q:
1955 blk_cleanup_queue(ctrl->fabrics_q);
1958 blk_mq_free_tag_set(ctrl->admin_tagset);
1960 nvme_tcp_free_admin_queue(ctrl);
1964 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
1967 blk_mq_quiesce_queue(ctrl->admin_q);
1968 blk_sync_queue(ctrl->admin_q);
1969 nvme_tcp_stop_queue(ctrl, 0);
1970 if (ctrl->admin_tagset) {
1971 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
1972 nvme_cancel_request, ctrl);
1973 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
1976 blk_mq_unquiesce_queue(ctrl->admin_q);
1977 nvme_tcp_destroy_admin_queue(ctrl, remove);
1980 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
1983 if (ctrl->queue_count <= 1)
1985 blk_mq_quiesce_queue(ctrl->admin_q);
1986 nvme_start_freeze(ctrl);
1987 nvme_stop_queues(ctrl);
1988 nvme_sync_io_queues(ctrl);
1989 nvme_tcp_stop_io_queues(ctrl);
1991 blk_mq_tagset_busy_iter(ctrl->tagset,
1992 nvme_cancel_request, ctrl);
1993 blk_mq_tagset_wait_completed_request(ctrl->tagset);
1996 nvme_start_queues(ctrl);
1997 nvme_tcp_destroy_io_queues(ctrl, remove);
2000 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2002 /* If we are resetting/deleting then do nothing */
2003 if (ctrl->state != NVME_CTRL_CONNECTING) {
2004 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
2005 ctrl->state == NVME_CTRL_LIVE);
2009 if (nvmf_should_reconnect(ctrl)) {
2010 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2011 ctrl->opts->reconnect_delay);
2012 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2013 ctrl->opts->reconnect_delay * HZ);
2015 dev_info(ctrl->device, "Removing controller...\n");
2016 nvme_delete_ctrl(ctrl);
2020 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2022 struct nvmf_ctrl_options *opts = ctrl->opts;
2025 ret = nvme_tcp_configure_admin_queue(ctrl, new);
2030 dev_err(ctrl->device, "icdoff is not supported!\n");
2034 if (opts->queue_size > ctrl->sqsize + 1)
2035 dev_warn(ctrl->device,
2036 "queue_size %zu > ctrl sqsize %u, clamping down\n",
2037 opts->queue_size, ctrl->sqsize + 1);
2039 if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2040 dev_warn(ctrl->device,
2041 "sqsize %u > ctrl maxcmd %u, clamping down\n",
2042 ctrl->sqsize + 1, ctrl->maxcmd);
2043 ctrl->sqsize = ctrl->maxcmd - 1;
2046 if (ctrl->queue_count > 1) {
2047 ret = nvme_tcp_configure_io_queues(ctrl, new);
2052 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2054 * state change failure is ok if we started ctrl delete,
2055 * unless we're during creation of a new controller to
2056 * avoid races with teardown flow.
2058 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2059 ctrl->state != NVME_CTRL_DELETING_NOIO);
2065 nvme_start_ctrl(ctrl);
2069 if (ctrl->queue_count > 1) {
2070 nvme_stop_queues(ctrl);
2071 nvme_sync_io_queues(ctrl);
2072 nvme_tcp_stop_io_queues(ctrl);
2073 nvme_cancel_tagset(ctrl);
2074 nvme_tcp_destroy_io_queues(ctrl, new);
2077 blk_mq_quiesce_queue(ctrl->admin_q);
2078 blk_sync_queue(ctrl->admin_q);
2079 nvme_tcp_stop_queue(ctrl, 0);
2080 nvme_cancel_admin_tagset(ctrl);
2081 nvme_tcp_destroy_admin_queue(ctrl, new);
2085 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2087 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2088 struct nvme_tcp_ctrl, connect_work);
2089 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2091 ++ctrl->nr_reconnects;
2093 if (nvme_tcp_setup_ctrl(ctrl, false))
2096 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2097 ctrl->nr_reconnects);
2099 ctrl->nr_reconnects = 0;
2104 dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2105 ctrl->nr_reconnects);
2106 nvme_tcp_reconnect_or_remove(ctrl);
2109 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2111 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2112 struct nvme_tcp_ctrl, err_work);
2113 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2115 nvme_stop_keep_alive(ctrl);
2116 flush_work(&ctrl->async_event_work);
2117 nvme_tcp_teardown_io_queues(ctrl, false);
2118 /* unquiesce to fail fast pending requests */
2119 nvme_start_queues(ctrl);
2120 nvme_tcp_teardown_admin_queue(ctrl, false);
2121 blk_mq_unquiesce_queue(ctrl->admin_q);
2123 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2124 /* state change failure is ok if we started ctrl delete */
2125 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2126 ctrl->state != NVME_CTRL_DELETING_NOIO);
2130 nvme_tcp_reconnect_or_remove(ctrl);
2133 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2135 nvme_tcp_teardown_io_queues(ctrl, shutdown);
2136 blk_mq_quiesce_queue(ctrl->admin_q);
2138 nvme_shutdown_ctrl(ctrl);
2140 nvme_disable_ctrl(ctrl);
2141 nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2144 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2146 nvme_tcp_teardown_ctrl(ctrl, true);
2149 static void nvme_reset_ctrl_work(struct work_struct *work)
2151 struct nvme_ctrl *ctrl =
2152 container_of(work, struct nvme_ctrl, reset_work);
2154 nvme_stop_ctrl(ctrl);
2155 nvme_tcp_teardown_ctrl(ctrl, false);
2157 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2158 /* state change failure is ok if we started ctrl delete */
2159 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2160 ctrl->state != NVME_CTRL_DELETING_NOIO);
2164 if (nvme_tcp_setup_ctrl(ctrl, false))
2170 ++ctrl->nr_reconnects;
2171 nvme_tcp_reconnect_or_remove(ctrl);
2174 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2176 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2177 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2180 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2182 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2184 if (list_empty(&ctrl->list))
2187 mutex_lock(&nvme_tcp_ctrl_mutex);
2188 list_del(&ctrl->list);
2189 mutex_unlock(&nvme_tcp_ctrl_mutex);
2191 nvmf_free_options(nctrl->opts);
2193 kfree(ctrl->queues);
2197 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2199 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2203 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2204 NVME_SGL_FMT_TRANSPORT_A;
2207 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2208 struct nvme_command *c, u32 data_len)
2210 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2212 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2213 sg->length = cpu_to_le32(data_len);
2214 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2217 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2220 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2223 sg->length = cpu_to_le32(data_len);
2224 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2225 NVME_SGL_FMT_TRANSPORT_A;
2228 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2230 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2231 struct nvme_tcp_queue *queue = &ctrl->queues[0];
2232 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2233 struct nvme_command *cmd = &pdu->cmd;
2234 u8 hdgst = nvme_tcp_hdgst_len(queue);
2236 memset(pdu, 0, sizeof(*pdu));
2237 pdu->hdr.type = nvme_tcp_cmd;
2238 if (queue->hdr_digest)
2239 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2240 pdu->hdr.hlen = sizeof(*pdu);
2241 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2243 cmd->common.opcode = nvme_admin_async_event;
2244 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2245 cmd->common.flags |= NVME_CMD_SGL_METABUF;
2246 nvme_tcp_set_sg_null(cmd);
2248 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2249 ctrl->async_req.offset = 0;
2250 ctrl->async_req.curr_bio = NULL;
2251 ctrl->async_req.data_len = 0;
2253 nvme_tcp_queue_request(&ctrl->async_req, true, true);
2256 static void nvme_tcp_complete_timed_out(struct request *rq)
2258 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2259 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2261 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2262 if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {
2263 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
2264 blk_mq_complete_request(rq);
2268 static enum blk_eh_timer_return
2269 nvme_tcp_timeout(struct request *rq, bool reserved)
2271 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2272 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2273 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2275 dev_warn(ctrl->device,
2276 "queue %d: timeout request %#x type %d\n",
2277 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2279 if (ctrl->state != NVME_CTRL_LIVE) {
2281 * If we are resetting, connecting or deleting we should
2282 * complete immediately because we may block controller
2283 * teardown or setup sequence
2284 * - ctrl disable/shutdown fabrics requests
2285 * - connect requests
2286 * - initialization admin requests
2287 * - I/O requests that entered after unquiescing and
2288 * the controller stopped responding
2290 * All other requests should be cancelled by the error
2291 * recovery work, so it's fine that we fail it here.
2293 nvme_tcp_complete_timed_out(rq);
2298 * LIVE state should trigger the normal error recovery which will
2299 * handle completing this request.
2301 nvme_tcp_error_recovery(ctrl);
2302 return BLK_EH_RESET_TIMER;
2305 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2308 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2309 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2310 struct nvme_command *c = &pdu->cmd;
2312 c->common.flags |= NVME_CMD_SGL_METABUF;
2314 if (!blk_rq_nr_phys_segments(rq))
2315 nvme_tcp_set_sg_null(c);
2316 else if (rq_data_dir(rq) == WRITE &&
2317 req->data_len <= nvme_tcp_inline_data_size(queue))
2318 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2320 nvme_tcp_set_sg_host_data(c, req->data_len);
2325 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2328 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2329 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2330 struct nvme_tcp_queue *queue = req->queue;
2331 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2334 ret = nvme_setup_cmd(ns, rq, &pdu->cmd);
2338 req->state = NVME_TCP_SEND_CMD_PDU;
2343 req->data_len = blk_rq_nr_phys_segments(rq) ?
2344 blk_rq_payload_bytes(rq) : 0;
2345 req->curr_bio = rq->bio;
2347 if (rq_data_dir(rq) == WRITE &&
2348 req->data_len <= nvme_tcp_inline_data_size(queue))
2349 req->pdu_len = req->data_len;
2350 else if (req->curr_bio)
2351 nvme_tcp_init_iter(req, READ);
2353 pdu->hdr.type = nvme_tcp_cmd;
2355 if (queue->hdr_digest)
2356 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2357 if (queue->data_digest && req->pdu_len) {
2358 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2359 ddgst = nvme_tcp_ddgst_len(queue);
2361 pdu->hdr.hlen = sizeof(*pdu);
2362 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2364 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2366 ret = nvme_tcp_map_data(queue, rq);
2367 if (unlikely(ret)) {
2368 nvme_cleanup_cmd(rq);
2369 dev_err(queue->ctrl->ctrl.device,
2370 "Failed to map data (%d)\n", ret);
2377 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2379 struct nvme_tcp_queue *queue = hctx->driver_data;
2381 if (!llist_empty(&queue->req_list))
2382 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2385 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2386 const struct blk_mq_queue_data *bd)
2388 struct nvme_ns *ns = hctx->queue->queuedata;
2389 struct nvme_tcp_queue *queue = hctx->driver_data;
2390 struct request *rq = bd->rq;
2391 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2392 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2395 if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2396 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
2398 ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2402 blk_mq_start_request(rq);
2404 nvme_tcp_queue_request(req, true, bd->last);
2409 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2411 struct nvme_tcp_ctrl *ctrl = set->driver_data;
2412 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2414 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2415 /* separate read/write queues */
2416 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2417 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2418 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2419 set->map[HCTX_TYPE_READ].nr_queues =
2420 ctrl->io_queues[HCTX_TYPE_READ];
2421 set->map[HCTX_TYPE_READ].queue_offset =
2422 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2424 /* shared read/write queues */
2425 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2426 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2427 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2428 set->map[HCTX_TYPE_READ].nr_queues =
2429 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2430 set->map[HCTX_TYPE_READ].queue_offset = 0;
2432 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2433 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2435 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2436 /* map dedicated poll queues only if we have queues left */
2437 set->map[HCTX_TYPE_POLL].nr_queues =
2438 ctrl->io_queues[HCTX_TYPE_POLL];
2439 set->map[HCTX_TYPE_POLL].queue_offset =
2440 ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2441 ctrl->io_queues[HCTX_TYPE_READ];
2442 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2445 dev_info(ctrl->ctrl.device,
2446 "mapped %d/%d/%d default/read/poll queues.\n",
2447 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2448 ctrl->io_queues[HCTX_TYPE_READ],
2449 ctrl->io_queues[HCTX_TYPE_POLL]);
2454 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
2456 struct nvme_tcp_queue *queue = hctx->driver_data;
2457 struct sock *sk = queue->sock->sk;
2459 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2462 set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2463 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2464 sk_busy_loop(sk, true);
2465 nvme_tcp_try_recv(queue);
2466 clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2467 return queue->nr_cqe;
2470 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2471 .queue_rq = nvme_tcp_queue_rq,
2472 .commit_rqs = nvme_tcp_commit_rqs,
2473 .complete = nvme_complete_rq,
2474 .init_request = nvme_tcp_init_request,
2475 .exit_request = nvme_tcp_exit_request,
2476 .init_hctx = nvme_tcp_init_hctx,
2477 .timeout = nvme_tcp_timeout,
2478 .map_queues = nvme_tcp_map_queues,
2479 .poll = nvme_tcp_poll,
2482 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2483 .queue_rq = nvme_tcp_queue_rq,
2484 .complete = nvme_complete_rq,
2485 .init_request = nvme_tcp_init_request,
2486 .exit_request = nvme_tcp_exit_request,
2487 .init_hctx = nvme_tcp_init_admin_hctx,
2488 .timeout = nvme_tcp_timeout,
2491 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2493 .module = THIS_MODULE,
2494 .flags = NVME_F_FABRICS,
2495 .reg_read32 = nvmf_reg_read32,
2496 .reg_read64 = nvmf_reg_read64,
2497 .reg_write32 = nvmf_reg_write32,
2498 .free_ctrl = nvme_tcp_free_ctrl,
2499 .submit_async_event = nvme_tcp_submit_async_event,
2500 .delete_ctrl = nvme_tcp_delete_ctrl,
2501 .get_address = nvmf_get_address,
2502 .stop_ctrl = nvme_tcp_stop_ctrl,
2506 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2508 struct nvme_tcp_ctrl *ctrl;
2511 mutex_lock(&nvme_tcp_ctrl_mutex);
2512 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2513 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2517 mutex_unlock(&nvme_tcp_ctrl_mutex);
2522 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2523 struct nvmf_ctrl_options *opts)
2525 struct nvme_tcp_ctrl *ctrl;
2528 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2530 return ERR_PTR(-ENOMEM);
2532 INIT_LIST_HEAD(&ctrl->list);
2533 ctrl->ctrl.opts = opts;
2534 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2535 opts->nr_poll_queues + 1;
2536 ctrl->ctrl.sqsize = opts->queue_size - 1;
2537 ctrl->ctrl.kato = opts->kato;
2539 INIT_DELAYED_WORK(&ctrl->connect_work,
2540 nvme_tcp_reconnect_ctrl_work);
2541 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2542 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2544 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2546 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2547 if (!opts->trsvcid) {
2551 opts->mask |= NVMF_OPT_TRSVCID;
2554 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2555 opts->traddr, opts->trsvcid, &ctrl->addr);
2557 pr_err("malformed address passed: %s:%s\n",
2558 opts->traddr, opts->trsvcid);
2562 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2563 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2564 opts->host_traddr, NULL, &ctrl->src_addr);
2566 pr_err("malformed src address passed: %s\n",
2572 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2577 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2579 if (!ctrl->queues) {
2584 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2586 goto out_kfree_queues;
2588 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2591 goto out_uninit_ctrl;
2594 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2596 goto out_uninit_ctrl;
2598 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2599 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2601 mutex_lock(&nvme_tcp_ctrl_mutex);
2602 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2603 mutex_unlock(&nvme_tcp_ctrl_mutex);
2608 nvme_uninit_ctrl(&ctrl->ctrl);
2609 nvme_put_ctrl(&ctrl->ctrl);
2612 return ERR_PTR(ret);
2614 kfree(ctrl->queues);
2617 return ERR_PTR(ret);
2620 static struct nvmf_transport_ops nvme_tcp_transport = {
2622 .module = THIS_MODULE,
2623 .required_opts = NVMF_OPT_TRADDR,
2624 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2625 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2626 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2627 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2629 .create_ctrl = nvme_tcp_create_ctrl,
2632 static int __init nvme_tcp_init_module(void)
2634 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2635 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2639 nvmf_register_transport(&nvme_tcp_transport);
2643 static void __exit nvme_tcp_cleanup_module(void)
2645 struct nvme_tcp_ctrl *ctrl;
2647 nvmf_unregister_transport(&nvme_tcp_transport);
2649 mutex_lock(&nvme_tcp_ctrl_mutex);
2650 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2651 nvme_delete_ctrl(&ctrl->ctrl);
2652 mutex_unlock(&nvme_tcp_ctrl_mutex);
2653 flush_workqueue(nvme_delete_wq);
2655 destroy_workqueue(nvme_tcp_wq);
2658 module_init(nvme_tcp_init_module);
2659 module_exit(nvme_tcp_cleanup_module);
2661 MODULE_LICENSE("GPL v2");