1 // SPDX-License-Identifier: GPL-2.0
3 * Ceph msgr2 protocol implementation
5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
8 #include <linux/ceph/ceph_debug.h>
10 #include <crypto/aead.h>
11 #include <crypto/algapi.h> /* for crypto_memneq() */
12 #include <crypto/hash.h>
13 #include <crypto/sha2.h>
14 #include <linux/bvec.h>
15 #include <linux/crc32c.h>
16 #include <linux/net.h>
17 #include <linux/scatterlist.h>
18 #include <linux/socket.h>
19 #include <linux/sched/mm.h>
23 #include <linux/ceph/ceph_features.h>
24 #include <linux/ceph/decode.h>
25 #include <linux/ceph/libceph.h>
26 #include <linux/ceph/messenger.h>
28 #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
30 #define FRAME_TAG_HELLO 1
31 #define FRAME_TAG_AUTH_REQUEST 2
32 #define FRAME_TAG_AUTH_BAD_METHOD 3
33 #define FRAME_TAG_AUTH_REPLY_MORE 4
34 #define FRAME_TAG_AUTH_REQUEST_MORE 5
35 #define FRAME_TAG_AUTH_DONE 6
36 #define FRAME_TAG_AUTH_SIGNATURE 7
37 #define FRAME_TAG_CLIENT_IDENT 8
38 #define FRAME_TAG_SERVER_IDENT 9
39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
40 #define FRAME_TAG_SESSION_RECONNECT 11
41 #define FRAME_TAG_SESSION_RESET 12
42 #define FRAME_TAG_SESSION_RETRY 13
43 #define FRAME_TAG_SESSION_RETRY_GLOBAL 14
44 #define FRAME_TAG_SESSION_RECONNECT_OK 15
45 #define FRAME_TAG_WAIT 16
46 #define FRAME_TAG_MESSAGE 17
47 #define FRAME_TAG_KEEPALIVE2 18
48 #define FRAME_TAG_KEEPALIVE2_ACK 19
49 #define FRAME_TAG_ACK 20
51 #define FRAME_LATE_STATUS_ABORTED 0x1
52 #define FRAME_LATE_STATUS_COMPLETE 0xe
53 #define FRAME_LATE_STATUS_ABORTED_MASK 0xf
55 #define IN_S_HANDLE_PREAMBLE 1
56 #define IN_S_HANDLE_CONTROL 2
57 #define IN_S_HANDLE_CONTROL_REMAINDER 3
58 #define IN_S_PREPARE_READ_DATA 4
59 #define IN_S_PREPARE_READ_DATA_CONT 5
60 #define IN_S_HANDLE_EPILOGUE 6
61 #define IN_S_FINISH_SKIP 7
63 #define OUT_S_QUEUE_DATA 1
64 #define OUT_S_QUEUE_DATA_CONT 2
65 #define OUT_S_QUEUE_ENC_PAGE 3
66 #define OUT_S_QUEUE_ZEROS 4
67 #define OUT_S_FINISH_MESSAGE 5
68 #define OUT_S_GET_NEXT 6
70 #define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
71 #define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
72 #define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
73 #define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
75 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
77 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
79 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
83 while (iov_iter_count(it)) {
84 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
91 iov_iter_advance(it, ret);
94 WARN_ON(msg_data_left(&msg));
99 * Read as much as possible.
102 * 1 - done, nothing (else) to read
103 * 0 - socket is empty, need to wait
106 static int ceph_tcp_recv(struct ceph_connection *con)
110 dout("%s con %p %s %zu\n", __func__, con,
111 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
112 iov_iter_count(&con->v2.in_iter));
113 ret = do_recvmsg(con->sock, &con->v2.in_iter);
114 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
115 iov_iter_count(&con->v2.in_iter));
119 static int do_sendmsg(struct socket *sock, struct iov_iter *it)
121 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
125 while (iov_iter_count(it)) {
126 ret = sock_sendmsg(sock, &msg);
133 iov_iter_advance(it, ret);
136 WARN_ON(msg_data_left(&msg));
140 static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
142 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
146 if (WARN_ON(!iov_iter_is_bvec(it)))
149 while (iov_iter_count(it)) {
150 /* iov_iter_iovec() for ITER_BVEC */
151 bv.bv_page = it->bvec->bv_page;
152 bv.bv_offset = it->bvec->bv_offset + it->iov_offset;
153 bv.bv_len = min(iov_iter_count(it),
154 it->bvec->bv_len - it->iov_offset);
157 * sendpage cannot properly handle pages with
158 * page_count == 0, we need to fall back to sendmsg if
161 * Same goes for slab pages: skb_can_coalesce() allows
162 * coalescing neighboring slab objects into a single frag
163 * which triggers one of hardened usercopy checks.
165 if (sendpage_ok(bv.bv_page)) {
166 ret = sock->ops->sendpage(sock, bv.bv_page,
167 bv.bv_offset, bv.bv_len,
170 iov_iter_bvec(&msg.msg_iter, WRITE, &bv, 1, bv.bv_len);
171 ret = sock_sendmsg(sock, &msg);
179 iov_iter_advance(it, ret);
186 * Write as much as possible. The socket is expected to be corked,
187 * so we don't bother with MSG_MORE/MSG_SENDPAGE_NOTLAST here.
190 * 1 - done, nothing (else) to write
191 * 0 - socket is full, need to wait
194 static int ceph_tcp_send(struct ceph_connection *con)
198 dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
199 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
200 if (con->v2.out_iter_sendpage)
201 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
203 ret = do_sendmsg(con->sock, &con->v2.out_iter);
204 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
205 iov_iter_count(&con->v2.out_iter));
209 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
211 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
212 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
214 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
215 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
216 con->v2.in_kvec_cnt++;
218 con->v2.in_iter.nr_segs++;
219 con->v2.in_iter.count += len;
222 static void reset_in_kvecs(struct ceph_connection *con)
224 WARN_ON(iov_iter_count(&con->v2.in_iter));
226 con->v2.in_kvec_cnt = 0;
227 iov_iter_kvec(&con->v2.in_iter, READ, con->v2.in_kvecs, 0, 0);
230 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
232 WARN_ON(iov_iter_count(&con->v2.in_iter));
234 con->v2.in_bvec = *bv;
235 iov_iter_bvec(&con->v2.in_iter, READ, &con->v2.in_bvec, 1, bv->bv_len);
238 static void set_in_skip(struct ceph_connection *con, int len)
240 WARN_ON(iov_iter_count(&con->v2.in_iter));
242 dout("%s con %p len %d\n", __func__, con, len);
243 iov_iter_discard(&con->v2.in_iter, READ, len);
246 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
248 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
249 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
250 WARN_ON(con->v2.out_zero);
252 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
253 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
254 con->v2.out_kvec_cnt++;
256 con->v2.out_iter.nr_segs++;
257 con->v2.out_iter.count += len;
260 static void reset_out_kvecs(struct ceph_connection *con)
262 WARN_ON(iov_iter_count(&con->v2.out_iter));
263 WARN_ON(con->v2.out_zero);
265 con->v2.out_kvec_cnt = 0;
267 iov_iter_kvec(&con->v2.out_iter, WRITE, con->v2.out_kvecs, 0, 0);
268 con->v2.out_iter_sendpage = false;
271 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
274 WARN_ON(iov_iter_count(&con->v2.out_iter));
275 WARN_ON(con->v2.out_zero);
277 con->v2.out_bvec = *bv;
278 con->v2.out_iter_sendpage = zerocopy;
279 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1,
280 con->v2.out_bvec.bv_len);
283 static void set_out_bvec_zero(struct ceph_connection *con)
285 WARN_ON(iov_iter_count(&con->v2.out_iter));
286 WARN_ON(!con->v2.out_zero);
288 con->v2.out_bvec.bv_page = ceph_zero_page;
289 con->v2.out_bvec.bv_offset = 0;
290 con->v2.out_bvec.bv_len = min(con->v2.out_zero, (int)PAGE_SIZE);
291 con->v2.out_iter_sendpage = true;
292 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1,
293 con->v2.out_bvec.bv_len);
296 static void out_zero_add(struct ceph_connection *con, int len)
298 dout("%s con %p len %d\n", __func__, con, len);
299 con->v2.out_zero += len;
302 static void *alloc_conn_buf(struct ceph_connection *con, int len)
306 dout("%s con %p len %d\n", __func__, con, len);
308 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
311 buf = ceph_kvmalloc(len, GFP_NOIO);
315 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
319 static void free_conn_bufs(struct ceph_connection *con)
321 while (con->v2.conn_buf_cnt)
322 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
325 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
327 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
329 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
331 con->v2.in_sign_kvec_cnt++;
334 static void clear_in_sign_kvecs(struct ceph_connection *con)
336 con->v2.in_sign_kvec_cnt = 0;
339 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
341 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
343 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
345 con->v2.out_sign_kvec_cnt++;
348 static void clear_out_sign_kvecs(struct ceph_connection *con)
350 con->v2.out_sign_kvec_cnt = 0;
353 static bool con_secure(struct ceph_connection *con)
355 return con->v2.con_mode == CEPH_CON_MODE_SECURE;
358 static int front_len(const struct ceph_msg *msg)
360 return le32_to_cpu(msg->hdr.front_len);
363 static int middle_len(const struct ceph_msg *msg)
365 return le32_to_cpu(msg->hdr.middle_len);
368 static int data_len(const struct ceph_msg *msg)
370 return le32_to_cpu(msg->hdr.data_len);
373 static bool need_padding(int len)
375 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
378 static int padded_len(int len)
380 return ALIGN(len, CEPH_GCM_BLOCK_LEN);
383 static int padding_len(int len)
385 return padded_len(len) - len;
388 /* preamble + control segment */
389 static int head_onwire_len(int ctrl_len, bool secure)
394 BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
397 head_len = CEPH_PREAMBLE_SECURE_LEN;
398 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
399 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
400 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
403 head_len = CEPH_PREAMBLE_PLAIN_LEN;
405 head_len += ctrl_len + CEPH_CRC_LEN;
410 /* front, middle and data segments + epilogue */
411 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
414 BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
415 middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
416 data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
418 if (!front_len && !middle_len && !data_len)
422 return front_len + middle_len + data_len +
423 CEPH_EPILOGUE_PLAIN_LEN;
425 return padded_len(front_len) + padded_len(middle_len) +
426 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
429 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
431 return __tail_onwire_len(front_len(msg), middle_len(msg),
432 data_len(msg), secure);
435 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
436 #define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
437 sizeof(struct ceph_msg_header2) + \
440 static const int frame_aligns[] = {
448 * Discards trailing empty segments, unless there is just one segment.
449 * A frame always has at least one (possibly empty) segment.
451 static int calc_segment_count(const int *lens, int len_cnt)
455 for (i = len_cnt - 1; i >= 0; i--) {
463 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
464 const int *lens, int len_cnt)
468 memset(desc, 0, sizeof(*desc));
471 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
472 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
473 for (i = 0; i < desc->fd_seg_cnt; i++) {
474 desc->fd_lens[i] = lens[i];
475 desc->fd_aligns[i] = frame_aligns[i];
480 * Preamble crc covers everything up to itself (28 bytes) and
481 * is calculated and verified irrespective of the connection mode
482 * (i.e. even if the frame is encrypted).
484 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
486 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
490 memset(p, 0, CEPH_PREAMBLE_LEN);
492 ceph_encode_8(&p, desc->fd_tag);
493 ceph_encode_8(&p, desc->fd_seg_cnt);
494 for (i = 0; i < desc->fd_seg_cnt; i++) {
495 ceph_encode_32(&p, desc->fd_lens[i]);
496 ceph_encode_16(&p, desc->fd_aligns[i]);
499 put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
502 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
504 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
505 u32 crc, expected_crc;
508 crc = crc32c(0, p, crcp - p);
509 expected_crc = get_unaligned_le32(crcp);
510 if (crc != expected_crc) {
511 pr_err("bad preamble crc, calculated %u, expected %u\n",
516 memset(desc, 0, sizeof(*desc));
518 desc->fd_tag = ceph_decode_8(&p);
519 desc->fd_seg_cnt = ceph_decode_8(&p);
520 if (desc->fd_seg_cnt < 1 ||
521 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
522 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
525 for (i = 0; i < desc->fd_seg_cnt; i++) {
526 desc->fd_lens[i] = ceph_decode_32(&p);
527 desc->fd_aligns[i] = ceph_decode_16(&p);
530 if (desc->fd_lens[0] < 0 ||
531 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
532 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
535 if (desc->fd_lens[1] < 0 ||
536 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
537 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
540 if (desc->fd_lens[2] < 0 ||
541 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
542 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
545 if (desc->fd_lens[3] < 0 ||
546 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
547 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
552 * This would fire for FRAME_TAG_WAIT (it has one empty
553 * segment), but we should never get it as client.
555 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
556 pr_err("last segment empty, segment count %d\n",
564 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
566 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
567 FRAME_LATE_STATUS_COMPLETE;
568 cpu_to_le32s(&con->v2.out_epil.front_crc);
569 cpu_to_le32s(&con->v2.out_epil.middle_crc);
570 cpu_to_le32s(&con->v2.out_epil.data_crc);
573 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
575 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
576 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
577 FRAME_LATE_STATUS_COMPLETE;
580 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
585 late_status = ceph_decode_8(&p);
586 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
587 FRAME_LATE_STATUS_COMPLETE) {
588 /* we should never get an aborted message as client */
589 pr_err("bad late_status 0x%x\n", late_status);
593 if (front_crc && middle_crc && data_crc) {
594 *front_crc = ceph_decode_32(&p);
595 *middle_crc = ceph_decode_32(&p);
596 *data_crc = ceph_decode_32(&p);
602 static void fill_header(struct ceph_msg_header *hdr,
603 const struct ceph_msg_header2 *hdr2,
604 int front_len, int middle_len, int data_len,
605 const struct ceph_entity_name *peer_name)
607 hdr->seq = hdr2->seq;
608 hdr->tid = hdr2->tid;
609 hdr->type = hdr2->type;
610 hdr->priority = hdr2->priority;
611 hdr->version = hdr2->version;
612 hdr->front_len = cpu_to_le32(front_len);
613 hdr->middle_len = cpu_to_le32(middle_len);
614 hdr->data_len = cpu_to_le32(data_len);
615 hdr->data_off = hdr2->data_off;
616 hdr->src = *peer_name;
617 hdr->compat_version = hdr2->compat_version;
622 static void fill_header2(struct ceph_msg_header2 *hdr2,
623 const struct ceph_msg_header *hdr, u64 ack_seq)
625 hdr2->seq = hdr->seq;
626 hdr2->tid = hdr->tid;
627 hdr2->type = hdr->type;
628 hdr2->priority = hdr->priority;
629 hdr2->version = hdr->version;
630 hdr2->data_pre_padding_len = 0;
631 hdr2->data_off = hdr->data_off;
632 hdr2->ack_seq = cpu_to_le64(ack_seq);
634 hdr2->compat_version = hdr->compat_version;
638 static int verify_control_crc(struct ceph_connection *con)
640 int ctrl_len = con->v2.in_desc.fd_lens[0];
641 u32 crc, expected_crc;
643 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
644 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
646 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
647 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
648 if (crc != expected_crc) {
649 pr_err("bad control crc, calculated %u, expected %u\n",
657 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
658 u32 middle_crc, u32 data_crc)
660 if (front_len(con->in_msg)) {
661 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
662 front_len(con->in_msg));
664 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
665 con->in_front_crc = -1;
668 if (middle_len(con->in_msg))
669 con->in_middle_crc = crc32c(-1,
670 con->in_msg->middle->vec.iov_base,
671 middle_len(con->in_msg));
672 else if (data_len(con->in_msg))
673 con->in_middle_crc = -1;
675 con->in_middle_crc = 0;
677 if (!data_len(con->in_msg))
678 con->in_data_crc = 0;
680 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
681 con->in_front_crc, con->in_middle_crc, con->in_data_crc);
683 if (con->in_front_crc != front_crc) {
684 pr_err("bad front crc, calculated %u, expected %u\n",
685 con->in_front_crc, front_crc);
688 if (con->in_middle_crc != middle_crc) {
689 pr_err("bad middle crc, calculated %u, expected %u\n",
690 con->in_middle_crc, middle_crc);
693 if (con->in_data_crc != data_crc) {
694 pr_err("bad data crc, calculated %u, expected %u\n",
695 con->in_data_crc, data_crc);
702 static int setup_crypto(struct ceph_connection *con,
703 const u8 *session_key, int session_key_len,
704 const u8 *con_secret, int con_secret_len)
706 unsigned int noio_flag;
709 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
710 __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
711 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
713 if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
714 con->v2.con_mode != CEPH_CON_MODE_SECURE) {
715 pr_err("bad con_mode %d\n", con->v2.con_mode);
719 if (!session_key_len) {
720 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
721 WARN_ON(con_secret_len);
722 return 0; /* auth_none */
725 noio_flag = memalloc_noio_save();
726 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
727 memalloc_noio_restore(noio_flag);
728 if (IS_ERR(con->v2.hmac_tfm)) {
729 ret = PTR_ERR(con->v2.hmac_tfm);
730 con->v2.hmac_tfm = NULL;
731 pr_err("failed to allocate hmac tfm context: %d\n", ret);
735 WARN_ON((unsigned long)session_key &
736 crypto_shash_alignmask(con->v2.hmac_tfm));
737 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
740 pr_err("failed to set hmac key: %d\n", ret);
744 if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
745 WARN_ON(con_secret_len);
746 return 0; /* auth_x, plain mode */
749 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
750 pr_err("con_secret too small %d\n", con_secret_len);
754 noio_flag = memalloc_noio_save();
755 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
756 memalloc_noio_restore(noio_flag);
757 if (IS_ERR(con->v2.gcm_tfm)) {
758 ret = PTR_ERR(con->v2.gcm_tfm);
759 con->v2.gcm_tfm = NULL;
760 pr_err("failed to allocate gcm tfm context: %d\n", ret);
764 WARN_ON((unsigned long)con_secret &
765 crypto_aead_alignmask(con->v2.gcm_tfm));
766 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
768 pr_err("failed to set gcm key: %d\n", ret);
772 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
773 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
775 pr_err("failed to set gcm tag size: %d\n", ret);
779 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
780 if (!con->v2.gcm_req) {
781 pr_err("failed to allocate gcm request\n");
785 crypto_init_wait(&con->v2.gcm_wait);
786 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
787 crypto_req_done, &con->v2.gcm_wait);
789 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
791 memcpy(&con->v2.out_gcm_nonce,
792 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
794 return 0; /* auth_x, secure mode */
797 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
798 int kvec_cnt, u8 *hmac)
800 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */
804 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
805 con->v2.hmac_tfm, kvec_cnt);
807 if (!con->v2.hmac_tfm) {
808 memset(hmac, 0, SHA256_DIGEST_SIZE);
809 return 0; /* auth_none */
812 desc->tfm = con->v2.hmac_tfm;
813 ret = crypto_shash_init(desc);
817 for (i = 0; i < kvec_cnt; i++) {
818 WARN_ON((unsigned long)kvecs[i].iov_base &
819 crypto_shash_alignmask(con->v2.hmac_tfm));
820 ret = crypto_shash_update(desc, kvecs[i].iov_base,
826 ret = crypto_shash_final(desc, hmac);
829 shash_desc_zero(desc);
830 return ret; /* auth_x, both plain and secure modes */
833 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
837 counter = le64_to_cpu(nonce->counter);
838 nonce->counter = cpu_to_le64(counter + 1);
841 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
842 struct scatterlist *src, struct scatterlist *dst,
845 struct ceph_gcm_nonce *nonce;
848 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
850 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */
851 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
852 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
853 crypto_aead_decrypt(con->v2.gcm_req),
858 gcm_inc_nonce(nonce);
862 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
868 WARN_ON(!cursor->total_resid);
870 /* skip zero-length data items */
871 while (!cursor->resid)
872 ceph_msg_data_advance(cursor, 0);
874 /* get a piece of data, cursor isn't advanced */
875 page = ceph_msg_data_next(cursor, &off, &len, NULL);
882 static int calc_sg_cnt(void *buf, int buf_len)
889 sg_cnt = need_padding(buf_len) ? 1 : 0;
890 if (is_vmalloc_addr(buf)) {
891 WARN_ON(offset_in_page(buf));
892 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
900 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
902 int data_len = cursor->total_resid;
909 sg_cnt = need_padding(data_len) ? 1 : 0;
911 get_bvec_at(cursor, &bv);
914 ceph_msg_data_advance(cursor, bv.bv_len);
915 } while (cursor->total_resid);
920 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
922 void *end = buf + buf_len;
930 if (is_vmalloc_addr(buf)) {
933 page = vmalloc_to_page(p);
934 len = min_t(int, end - p, PAGE_SIZE);
935 WARN_ON(!page || !len || offset_in_page(p));
936 sg_set_page(*sg, page, len, 0);
941 sg_set_buf(*sg, buf, buf_len);
945 if (need_padding(buf_len)) {
946 sg_set_buf(*sg, pad, padding_len(buf_len));
951 static void init_sgs_cursor(struct scatterlist **sg,
952 struct ceph_msg_data_cursor *cursor, u8 *pad)
954 int data_len = cursor->total_resid;
961 get_bvec_at(cursor, &bv);
962 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
965 ceph_msg_data_advance(cursor, bv.bv_len);
966 } while (cursor->total_resid);
968 if (need_padding(data_len)) {
969 sg_set_buf(*sg, pad, padding_len(data_len));
974 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
975 u8 *front_pad, u8 *middle_pad, u8 *data_pad,
976 void *epilogue, bool add_tag)
978 struct ceph_msg_data_cursor cursor;
979 struct scatterlist *cur_sg;
983 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
986 sg_cnt = 1; /* epilogue + [auth tag] */
988 sg_cnt += calc_sg_cnt(msg->front.iov_base,
991 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
994 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
995 sg_cnt += calc_sg_cnt_cursor(&cursor);
998 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1004 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1006 if (middle_len(msg))
1007 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1009 if (data_len(msg)) {
1010 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
1011 init_sgs_cursor(&cur_sg, &cursor, data_pad);
1014 WARN_ON(!sg_is_last(cur_sg));
1015 sg_set_buf(cur_sg, epilogue,
1016 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1020 static int decrypt_preamble(struct ceph_connection *con)
1022 struct scatterlist sg;
1024 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1025 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1028 static int decrypt_control_remainder(struct ceph_connection *con)
1030 int ctrl_len = con->v2.in_desc.fd_lens[0];
1031 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1032 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1033 struct scatterlist sgs[2];
1035 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1036 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1038 sg_init_table(sgs, 2);
1039 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1040 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1042 return gcm_crypt(con, false, sgs, sgs,
1043 padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1046 static int decrypt_message(struct ceph_connection *con)
1048 struct sg_table sgt = {};
1051 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1052 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1053 con->v2.in_buf, true);
1057 ret = gcm_crypt(con, false, sgt.sgl, sgt.sgl,
1058 tail_onwire_len(con->in_msg, true));
1061 sg_free_table(&sgt);
1065 static int prepare_banner(struct ceph_connection *con)
1067 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1070 buf = alloc_conn_buf(con, buf_len);
1075 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1076 ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1077 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1078 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1079 WARN_ON(p != buf + buf_len);
1081 add_out_kvec(con, buf, buf_len);
1082 add_out_sign_kvec(con, buf, buf_len);
1083 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1090 * control body (ctrl_len bytes)
1091 * space for control crc
1093 * extdata (optional):
1094 * control body (extdata_len bytes)
1096 * Compute control crc and gather base and extdata into:
1099 * control body (ctrl_len + extdata_len bytes)
1102 * Preamble should already be encoded at the start of base.
1104 static void prepare_head_plain(struct ceph_connection *con, void *base,
1105 int ctrl_len, void *extdata, int extdata_len,
1108 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1109 void *crcp = base + base_len - CEPH_CRC_LEN;
1112 crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1114 crc = crc32c(crc, extdata, extdata_len);
1115 put_unaligned_le32(crc, crcp);
1118 add_out_kvec(con, base, base_len);
1120 add_out_sign_kvec(con, base, base_len);
1124 add_out_kvec(con, base, crcp - base);
1125 add_out_kvec(con, extdata, extdata_len);
1126 add_out_kvec(con, crcp, CEPH_CRC_LEN);
1128 add_out_sign_kvec(con, base, crcp - base);
1129 add_out_sign_kvec(con, extdata, extdata_len);
1130 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1134 static int prepare_head_secure_small(struct ceph_connection *con,
1135 void *base, int ctrl_len)
1137 struct scatterlist sg;
1140 /* inline buffer padding? */
1141 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1142 memset(CTRL_BODY(base) + ctrl_len, 0,
1143 CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1145 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1146 ret = gcm_crypt(con, true, &sg, &sg,
1147 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1151 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1158 * control body (ctrl_len bytes)
1159 * space for padding, if needed
1160 * space for control remainder auth tag
1161 * space for preamble auth tag
1163 * Encrypt preamble and the inline portion, then encrypt the remainder
1167 * control body (48 bytes)
1169 * control body (ctrl_len - 48 bytes)
1170 * zero padding, if needed
1171 * control remainder auth tag
1173 * Preamble should already be encoded at the start of base.
1175 static int prepare_head_secure_big(struct ceph_connection *con,
1176 void *base, int ctrl_len)
1178 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1179 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1180 void *rem_tag = rem + padded_len(rem_len);
1181 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1182 struct scatterlist sgs[2];
1185 sg_init_table(sgs, 2);
1186 sg_set_buf(&sgs[0], base, rem - base);
1187 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1188 ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1192 /* control remainder padding? */
1193 if (need_padding(rem_len))
1194 memset(rem + rem_len, 0, padding_len(rem_len));
1196 sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1197 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1201 add_out_kvec(con, base, rem - base);
1202 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1203 add_out_kvec(con, rem, pmbl_tag - rem);
1207 static int __prepare_control(struct ceph_connection *con, int tag,
1208 void *base, int ctrl_len, void *extdata,
1209 int extdata_len, bool to_be_signed)
1211 int total_len = ctrl_len + extdata_len;
1212 struct ceph_frame_desc desc;
1215 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1216 total_len, ctrl_len, extdata_len);
1218 /* extdata may be vmalloc'ed but not base */
1219 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1222 init_frame_desc(&desc, tag, &total_len, 1);
1223 encode_preamble(&desc, base);
1225 if (con_secure(con)) {
1226 if (WARN_ON(extdata_len || to_be_signed))
1229 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1230 /* fully inlined, inline buffer may need padding */
1231 ret = prepare_head_secure_small(con, base, ctrl_len);
1233 /* partially inlined, inline buffer is full */
1234 ret = prepare_head_secure_big(con, base, ctrl_len);
1238 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1242 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1246 static int prepare_control(struct ceph_connection *con, int tag,
1247 void *base, int ctrl_len)
1249 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1252 static int prepare_hello(struct ceph_connection *con)
1257 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1258 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1263 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1264 ceph_encode_entity_addr(&p, &con->peer_addr);
1265 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1267 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1271 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1272 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1274 static int prepare_auth_request(struct ceph_connection *con)
1276 void *authorizer, *authorizer_copy;
1277 int ctrl_len, authorizer_len;
1281 ctrl_len = AUTH_BUF_LEN;
1282 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1286 mutex_unlock(&con->mutex);
1287 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1288 &authorizer, &authorizer_len);
1289 mutex_lock(&con->mutex);
1290 if (con->state != CEPH_CON_S_V2_HELLO) {
1291 dout("%s con %p state changed to %d\n", __func__, con,
1296 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1300 authorizer_copy = alloc_conn_buf(con, authorizer_len);
1301 if (!authorizer_copy)
1304 memcpy(authorizer_copy, authorizer, authorizer_len);
1306 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1307 authorizer_copy, authorizer_len, true);
1310 static int prepare_auth_request_more(struct ceph_connection *con,
1311 void *reply, int reply_len)
1313 int ctrl_len, authorizer_len;
1318 ctrl_len = AUTH_BUF_LEN;
1319 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1323 mutex_unlock(&con->mutex);
1324 ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1325 CTRL_BODY(buf), &ctrl_len,
1326 &authorizer, &authorizer_len);
1327 mutex_lock(&con->mutex);
1328 if (con->state != CEPH_CON_S_V2_AUTH) {
1329 dout("%s con %p state changed to %d\n", __func__, con,
1334 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1338 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1339 ctrl_len, authorizer, authorizer_len, true);
1342 static int prepare_auth_signature(struct ceph_connection *con)
1347 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1352 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1357 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1358 SHA256_DIGEST_SIZE);
1361 static int prepare_client_ident(struct ceph_connection *con)
1363 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1364 struct ceph_client *client = from_msgr(con->msgr);
1365 u64 global_id = ceph_client_gid(client);
1369 WARN_ON(con->v2.server_cookie);
1370 WARN_ON(con->v2.connect_seq);
1371 WARN_ON(con->v2.peer_global_seq);
1373 if (!con->v2.client_cookie) {
1375 get_random_bytes(&con->v2.client_cookie,
1376 sizeof(con->v2.client_cookie));
1377 } while (!con->v2.client_cookie);
1378 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1379 con->v2.client_cookie);
1381 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1382 con->v2.client_cookie);
1385 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1386 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1387 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1388 global_id, con->v2.global_seq, client->supported_features,
1389 client->required_features, con->v2.client_cookie);
1391 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1392 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1393 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1398 ceph_encode_8(&p, 2); /* addrvec marker */
1399 ceph_encode_32(&p, 1); /* addr_cnt */
1400 ceph_encode_entity_addr(&p, my_addr);
1401 ceph_encode_entity_addr(&p, &con->peer_addr);
1402 ceph_encode_64(&p, global_id);
1403 ceph_encode_64(&p, con->v2.global_seq);
1404 ceph_encode_64(&p, client->supported_features);
1405 ceph_encode_64(&p, client->required_features);
1406 ceph_encode_64(&p, 0); /* flags */
1407 ceph_encode_64(&p, con->v2.client_cookie);
1408 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1410 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1413 static int prepare_session_reconnect(struct ceph_connection *con)
1415 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1419 WARN_ON(!con->v2.client_cookie);
1420 WARN_ON(!con->v2.server_cookie);
1421 WARN_ON(!con->v2.connect_seq);
1422 WARN_ON(!con->v2.peer_global_seq);
1424 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1425 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1426 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1427 con->v2.connect_seq, con->in_seq);
1429 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1430 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1435 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */
1436 ceph_encode_32(&p, 1); /* my_addrs len */
1437 ceph_encode_entity_addr(&p, my_addr);
1438 ceph_encode_64(&p, con->v2.client_cookie);
1439 ceph_encode_64(&p, con->v2.server_cookie);
1440 ceph_encode_64(&p, con->v2.global_seq);
1441 ceph_encode_64(&p, con->v2.connect_seq);
1442 ceph_encode_64(&p, con->in_seq);
1443 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1445 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1448 static int prepare_keepalive2(struct ceph_connection *con)
1450 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1451 struct timespec64 now;
1453 ktime_get_real_ts64(&now);
1454 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1457 ceph_encode_timespec64(ts, &now);
1459 reset_out_kvecs(con);
1460 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1461 sizeof(struct ceph_timespec));
1464 static int prepare_ack(struct ceph_connection *con)
1468 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1469 con->in_seq_acked, con->in_seq);
1470 con->in_seq_acked = con->in_seq;
1472 p = CTRL_BODY(con->v2.out_buf);
1473 ceph_encode_64(&p, con->in_seq_acked);
1475 reset_out_kvecs(con);
1476 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1479 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1481 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1482 con->out_msg, aborted, con->v2.out_epil.front_crc,
1483 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1485 encode_epilogue_plain(con, aborted);
1486 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1490 * For "used" empty segments, crc is -1. For unused (trailing)
1491 * segments, crc is 0.
1493 static void prepare_message_plain(struct ceph_connection *con)
1495 struct ceph_msg *msg = con->out_msg;
1497 prepare_head_plain(con, con->v2.out_buf,
1498 sizeof(struct ceph_msg_header2), NULL, 0, false);
1500 if (!front_len(msg) && !middle_len(msg)) {
1501 if (!data_len(msg)) {
1503 * Empty message: once the head is written,
1504 * we are done -- there is no epilogue.
1506 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1510 con->v2.out_epil.front_crc = -1;
1511 con->v2.out_epil.middle_crc = -1;
1512 con->v2.out_state = OUT_S_QUEUE_DATA;
1516 if (front_len(msg)) {
1517 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1519 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1521 /* middle (at least) is there, checked above */
1522 con->v2.out_epil.front_crc = -1;
1525 if (middle_len(msg)) {
1526 con->v2.out_epil.middle_crc =
1527 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1528 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1530 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1533 if (data_len(msg)) {
1534 con->v2.out_state = OUT_S_QUEUE_DATA;
1536 con->v2.out_epil.data_crc = 0;
1537 prepare_epilogue_plain(con, false);
1538 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1543 * Unfortunately the kernel crypto API doesn't support streaming
1544 * (piecewise) operation for AEAD algorithms, so we can't get away
1545 * with a fixed size buffer and a couple sgs. Instead, we have to
1546 * allocate pages for the entire tail of the message (currently up
1547 * to ~32M) and two sgs arrays (up to ~256K each)...
1549 static int prepare_message_secure(struct ceph_connection *con)
1551 void *zerop = page_address(ceph_zero_page);
1552 struct sg_table enc_sgt = {};
1553 struct sg_table sgt = {};
1554 struct page **enc_pages;
1559 ret = prepare_head_secure_small(con, con->v2.out_buf,
1560 sizeof(struct ceph_msg_header2));
1564 tail_len = tail_onwire_len(con->out_msg, true);
1567 * Empty message: once the head is written,
1568 * we are done -- there is no epilogue.
1570 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1574 encode_epilogue_secure(con, false);
1575 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1576 &con->v2.out_epil, false);
1580 enc_page_cnt = calc_pages_for(0, tail_len);
1581 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1582 if (IS_ERR(enc_pages)) {
1583 ret = PTR_ERR(enc_pages);
1587 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1588 con->v2.out_enc_pages = enc_pages;
1589 con->v2.out_enc_page_cnt = enc_page_cnt;
1590 con->v2.out_enc_resid = tail_len;
1591 con->v2.out_enc_i = 0;
1593 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1594 0, tail_len, GFP_NOIO);
1598 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1599 tail_len - CEPH_GCM_TAG_LEN);
1603 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1604 con->out_msg, sgt.orig_nents, enc_page_cnt);
1605 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1608 sg_free_table(&sgt);
1609 sg_free_table(&enc_sgt);
1613 static int prepare_message(struct ceph_connection *con)
1616 sizeof(struct ceph_msg_header2),
1617 front_len(con->out_msg),
1618 middle_len(con->out_msg),
1619 data_len(con->out_msg)
1621 struct ceph_frame_desc desc;
1624 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1625 con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1627 if (con->in_seq > con->in_seq_acked) {
1628 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1629 con->in_seq_acked, con->in_seq);
1630 con->in_seq_acked = con->in_seq;
1633 reset_out_kvecs(con);
1634 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1635 encode_preamble(&desc, con->v2.out_buf);
1636 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1639 if (con_secure(con)) {
1640 ret = prepare_message_secure(con);
1644 prepare_message_plain(con);
1647 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1651 static int prepare_read_banner_prefix(struct ceph_connection *con)
1655 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1659 reset_in_kvecs(con);
1660 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1661 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1662 con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1666 static int prepare_read_banner_payload(struct ceph_connection *con,
1671 buf = alloc_conn_buf(con, payload_len);
1675 reset_in_kvecs(con);
1676 add_in_kvec(con, buf, payload_len);
1677 add_in_sign_kvec(con, buf, payload_len);
1678 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1682 static void prepare_read_preamble(struct ceph_connection *con)
1684 reset_in_kvecs(con);
1685 add_in_kvec(con, con->v2.in_buf,
1686 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1687 CEPH_PREAMBLE_PLAIN_LEN);
1688 con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1691 static int prepare_read_control(struct ceph_connection *con)
1693 int ctrl_len = con->v2.in_desc.fd_lens[0];
1697 reset_in_kvecs(con);
1698 if (con->state == CEPH_CON_S_V2_HELLO ||
1699 con->state == CEPH_CON_S_V2_AUTH) {
1700 head_len = head_onwire_len(ctrl_len, false);
1701 buf = alloc_conn_buf(con, head_len);
1705 /* preserve preamble */
1706 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1708 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1709 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1710 add_in_sign_kvec(con, buf, head_len);
1712 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1713 buf = alloc_conn_buf(con, ctrl_len);
1717 add_in_kvec(con, buf, ctrl_len);
1719 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1721 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1723 con->v2.in_state = IN_S_HANDLE_CONTROL;
1727 static int prepare_read_control_remainder(struct ceph_connection *con)
1729 int ctrl_len = con->v2.in_desc.fd_lens[0];
1730 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1733 buf = alloc_conn_buf(con, ctrl_len);
1737 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1739 reset_in_kvecs(con);
1740 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1741 add_in_kvec(con, con->v2.in_buf,
1742 padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1743 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1747 static void prepare_read_data(struct ceph_connection *con)
1751 if (!con_secure(con))
1752 con->in_data_crc = -1;
1753 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1754 data_len(con->in_msg));
1756 get_bvec_at(&con->v2.in_cursor, &bv);
1757 set_in_bvec(con, &bv);
1758 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1761 static void prepare_read_data_cont(struct ceph_connection *con)
1765 if (!con_secure(con))
1766 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1767 con->v2.in_bvec.bv_page,
1768 con->v2.in_bvec.bv_offset,
1769 con->v2.in_bvec.bv_len);
1771 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1772 if (con->v2.in_cursor.total_resid) {
1773 get_bvec_at(&con->v2.in_cursor, &bv);
1774 set_in_bvec(con, &bv);
1775 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1780 * We've read all data. Prepare to read data padding (if any)
1783 reset_in_kvecs(con);
1784 if (con_secure(con)) {
1785 if (need_padding(data_len(con->in_msg)))
1786 add_in_kvec(con, DATA_PAD(con->v2.in_buf),
1787 padding_len(data_len(con->in_msg)));
1788 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_SECURE_LEN);
1790 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1792 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1795 static void __finish_skip(struct ceph_connection *con)
1798 prepare_read_preamble(con);
1801 static void prepare_skip_message(struct ceph_connection *con)
1803 struct ceph_frame_desc *desc = &con->v2.in_desc;
1806 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
1807 desc->fd_lens[2], desc->fd_lens[3]);
1809 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
1810 desc->fd_lens[3], con_secure(con));
1814 set_in_skip(con, tail_len);
1815 con->v2.in_state = IN_S_FINISH_SKIP;
1819 static int process_banner_prefix(struct ceph_connection *con)
1824 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
1826 p = con->v2.in_kvecs[0].iov_base;
1827 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
1828 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
1829 con->error_msg = "server is speaking msgr1 protocol";
1831 con->error_msg = "protocol error, bad banner";
1835 p += CEPH_BANNER_V2_LEN;
1836 payload_len = ceph_decode_16(&p);
1837 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
1839 return prepare_read_banner_payload(con, payload_len);
1842 static int process_banner_payload(struct ceph_connection *con)
1844 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
1845 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
1846 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
1847 u64 server_feat, server_req_feat;
1851 p = con->v2.in_kvecs[0].iov_base;
1852 ceph_decode_64_safe(&p, end, server_feat, bad);
1853 ceph_decode_64_safe(&p, end, server_req_feat, bad);
1855 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
1856 __func__, con, server_feat, server_req_feat);
1858 if (req_feat & ~server_feat) {
1859 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
1860 server_feat, req_feat & ~server_feat);
1861 con->error_msg = "missing required protocol features";
1864 if (server_req_feat & ~feat) {
1865 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
1866 feat, server_req_feat & ~feat);
1867 con->error_msg = "missing required protocol features";
1871 /* no reset_out_kvecs() as our banner may still be pending */
1872 ret = prepare_hello(con);
1874 pr_err("prepare_hello failed: %d\n", ret);
1878 con->state = CEPH_CON_S_V2_HELLO;
1879 prepare_read_preamble(con);
1883 pr_err("failed to decode banner payload\n");
1887 static int process_hello(struct ceph_connection *con, void *p, void *end)
1889 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1890 struct ceph_entity_addr addr_for_me;
1894 if (con->state != CEPH_CON_S_V2_HELLO) {
1895 con->error_msg = "protocol error, unexpected hello";
1899 ceph_decode_8_safe(&p, end, entity_type, bad);
1900 ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
1902 pr_err("failed to decode addr_for_me: %d\n", ret);
1906 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
1907 entity_type, ceph_pr_addr(&addr_for_me));
1909 if (entity_type != con->peer_name.type) {
1910 pr_err("bad peer type, want %d, got %d\n",
1911 con->peer_name.type, entity_type);
1912 con->error_msg = "wrong peer at address";
1917 * Set our address to the address our first peer (i.e. monitor)
1918 * sees that we are connecting from. If we are behind some sort
1919 * of NAT and want to be identified by some private (not NATed)
1920 * address, ip option should be used.
1922 if (ceph_addr_is_blank(my_addr)) {
1923 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
1924 sizeof(my_addr->in_addr));
1925 ceph_addr_set_port(my_addr, 0);
1926 dout("%s con %p set my addr %s, as seen by peer %s\n",
1927 __func__, con, ceph_pr_addr(my_addr),
1928 ceph_pr_addr(&con->peer_addr));
1930 dout("%s con %p my addr already set %s\n",
1931 __func__, con, ceph_pr_addr(my_addr));
1934 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
1935 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
1936 WARN_ON(!my_addr->nonce);
1938 /* no reset_out_kvecs() as our hello may still be pending */
1939 ret = prepare_auth_request(con);
1942 pr_err("prepare_auth_request failed: %d\n", ret);
1946 con->state = CEPH_CON_S_V2_AUTH;
1950 pr_err("failed to decode hello\n");
1954 static int process_auth_bad_method(struct ceph_connection *con,
1957 int allowed_protos[8], allowed_modes[8];
1958 int allowed_proto_cnt, allowed_mode_cnt;
1959 int used_proto, result;
1963 if (con->state != CEPH_CON_S_V2_AUTH) {
1964 con->error_msg = "protocol error, unexpected auth_bad_method";
1968 ceph_decode_32_safe(&p, end, used_proto, bad);
1969 ceph_decode_32_safe(&p, end, result, bad);
1970 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
1973 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
1974 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
1975 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
1978 for (i = 0; i < allowed_proto_cnt; i++) {
1979 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
1980 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
1981 i, allowed_protos[i]);
1984 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
1985 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
1986 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
1989 for (i = 0; i < allowed_mode_cnt; i++) {
1990 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
1991 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
1992 i, allowed_modes[i]);
1995 mutex_unlock(&con->mutex);
1996 ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2001 mutex_lock(&con->mutex);
2002 if (con->state != CEPH_CON_S_V2_AUTH) {
2003 dout("%s con %p state changed to %d\n", __func__, con,
2008 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2012 pr_err("failed to decode auth_bad_method\n");
2016 static int process_auth_reply_more(struct ceph_connection *con,
2022 if (con->state != CEPH_CON_S_V2_AUTH) {
2023 con->error_msg = "protocol error, unexpected auth_reply_more";
2027 ceph_decode_32_safe(&p, end, payload_len, bad);
2028 ceph_decode_need(&p, end, payload_len, bad);
2030 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2032 reset_out_kvecs(con);
2033 ret = prepare_auth_request_more(con, p, payload_len);
2036 pr_err("prepare_auth_request_more failed: %d\n", ret);
2043 pr_err("failed to decode auth_reply_more\n");
2048 * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2049 * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2050 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack
2051 * objects, so do it by hand.
2053 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2055 u8 session_key_buf[CEPH_KEY_LEN + 16];
2056 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2057 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2058 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2059 int session_key_len, con_secret_len;
2064 if (con->state != CEPH_CON_S_V2_AUTH) {
2065 con->error_msg = "protocol error, unexpected auth_done";
2069 ceph_decode_64_safe(&p, end, global_id, bad);
2070 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2071 ceph_decode_32_safe(&p, end, payload_len, bad);
2073 dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2074 __func__, con, global_id, con->v2.con_mode, payload_len);
2076 mutex_unlock(&con->mutex);
2077 session_key_len = 0;
2079 ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2080 session_key, &session_key_len,
2081 con_secret, &con_secret_len);
2082 mutex_lock(&con->mutex);
2083 if (con->state != CEPH_CON_S_V2_AUTH) {
2084 dout("%s con %p state changed to %d\n", __func__, con,
2090 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2094 ret = setup_crypto(con, session_key, session_key_len, con_secret,
2099 reset_out_kvecs(con);
2100 ret = prepare_auth_signature(con);
2102 pr_err("prepare_auth_signature failed: %d\n", ret);
2106 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2109 memzero_explicit(session_key_buf, sizeof(session_key_buf));
2110 memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2114 pr_err("failed to decode auth_done\n");
2118 static int process_auth_signature(struct ceph_connection *con,
2121 u8 hmac[SHA256_DIGEST_SIZE];
2124 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2125 con->error_msg = "protocol error, unexpected auth_signature";
2129 ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2130 con->v2.out_sign_kvec_cnt, hmac);
2134 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2135 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2136 con->error_msg = "integrity error, bad auth signature";
2140 dout("%s con %p auth signature ok\n", __func__, con);
2142 /* no reset_out_kvecs() as our auth_signature may still be pending */
2143 if (!con->v2.server_cookie) {
2144 ret = prepare_client_ident(con);
2146 pr_err("prepare_client_ident failed: %d\n", ret);
2150 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2152 ret = prepare_session_reconnect(con);
2154 pr_err("prepare_session_reconnect failed: %d\n", ret);
2158 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2164 pr_err("failed to decode auth_signature\n");
2168 static int process_server_ident(struct ceph_connection *con,
2171 struct ceph_client *client = from_msgr(con->msgr);
2172 u64 features, required_features;
2173 struct ceph_entity_addr addr;
2180 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2181 con->error_msg = "protocol error, unexpected server_ident";
2185 ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2187 pr_err("failed to decode server addrs: %d\n", ret);
2191 ceph_decode_64_safe(&p, end, global_id, bad);
2192 ceph_decode_64_safe(&p, end, global_seq, bad);
2193 ceph_decode_64_safe(&p, end, features, bad);
2194 ceph_decode_64_safe(&p, end, required_features, bad);
2195 ceph_decode_64_safe(&p, end, flags, bad);
2196 ceph_decode_64_safe(&p, end, cookie, bad);
2198 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2199 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2200 global_id, global_seq, features, required_features, flags, cookie);
2202 /* is this who we intended to talk to? */
2203 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2204 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2205 ceph_pr_addr(&con->peer_addr),
2206 le32_to_cpu(con->peer_addr.nonce),
2207 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2208 con->error_msg = "wrong peer at address";
2212 if (client->required_features & ~features) {
2213 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2214 features, client->required_features & ~features);
2215 con->error_msg = "missing required protocol features";
2220 * Both name->type and name->num are set in ceph_con_open() but
2221 * name->num may be bogus in the initial monmap. name->type is
2222 * verified in handle_hello().
2224 WARN_ON(!con->peer_name.type);
2225 con->peer_name.num = cpu_to_le64(global_id);
2226 con->v2.peer_global_seq = global_seq;
2227 con->peer_features = features;
2228 WARN_ON(required_features & ~client->supported_features);
2229 con->v2.server_cookie = cookie;
2231 if (flags & CEPH_MSG_CONNECT_LOSSY) {
2232 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2233 WARN_ON(con->v2.server_cookie);
2235 WARN_ON(!con->v2.server_cookie);
2238 clear_in_sign_kvecs(con);
2239 clear_out_sign_kvecs(con);
2240 free_conn_bufs(con);
2241 con->delay = 0; /* reset backoff memory */
2243 con->state = CEPH_CON_S_OPEN;
2244 con->v2.out_state = OUT_S_GET_NEXT;
2248 pr_err("failed to decode server_ident\n");
2252 static int process_ident_missing_features(struct ceph_connection *con,
2255 struct ceph_client *client = from_msgr(con->msgr);
2256 u64 missing_features;
2258 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2259 con->error_msg = "protocol error, unexpected ident_missing_features";
2263 ceph_decode_64_safe(&p, end, missing_features, bad);
2264 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2265 client->supported_features, missing_features);
2266 con->error_msg = "missing required protocol features";
2270 pr_err("failed to decode ident_missing_features\n");
2274 static int process_session_reconnect_ok(struct ceph_connection *con,
2279 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2280 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2284 ceph_decode_64_safe(&p, end, seq, bad);
2286 dout("%s con %p seq %llu\n", __func__, con, seq);
2287 ceph_con_discard_requeued(con, seq);
2289 clear_in_sign_kvecs(con);
2290 clear_out_sign_kvecs(con);
2291 free_conn_bufs(con);
2292 con->delay = 0; /* reset backoff memory */
2294 con->state = CEPH_CON_S_OPEN;
2295 con->v2.out_state = OUT_S_GET_NEXT;
2299 pr_err("failed to decode session_reconnect_ok\n");
2303 static int process_session_retry(struct ceph_connection *con,
2309 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2310 con->error_msg = "protocol error, unexpected session_retry";
2314 ceph_decode_64_safe(&p, end, connect_seq, bad);
2316 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2317 WARN_ON(connect_seq <= con->v2.connect_seq);
2318 con->v2.connect_seq = connect_seq + 1;
2320 free_conn_bufs(con);
2322 reset_out_kvecs(con);
2323 ret = prepare_session_reconnect(con);
2325 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2332 pr_err("failed to decode session_retry\n");
2336 static int process_session_retry_global(struct ceph_connection *con,
2342 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2343 con->error_msg = "protocol error, unexpected session_retry_global";
2347 ceph_decode_64_safe(&p, end, global_seq, bad);
2349 dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2350 WARN_ON(global_seq <= con->v2.global_seq);
2351 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2353 free_conn_bufs(con);
2355 reset_out_kvecs(con);
2356 ret = prepare_session_reconnect(con);
2358 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2365 pr_err("failed to decode session_retry_global\n");
2369 static int process_session_reset(struct ceph_connection *con,
2375 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2376 con->error_msg = "protocol error, unexpected session_reset";
2380 ceph_decode_8_safe(&p, end, full, bad);
2382 con->error_msg = "protocol error, bad session_reset";
2386 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2387 ceph_pr_addr(&con->peer_addr));
2388 ceph_con_reset_session(con);
2390 mutex_unlock(&con->mutex);
2391 if (con->ops->peer_reset)
2392 con->ops->peer_reset(con);
2393 mutex_lock(&con->mutex);
2394 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2395 dout("%s con %p state changed to %d\n", __func__, con,
2400 free_conn_bufs(con);
2402 reset_out_kvecs(con);
2403 ret = prepare_client_ident(con);
2405 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2409 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2413 pr_err("failed to decode session_reset\n");
2417 static int process_keepalive2_ack(struct ceph_connection *con,
2420 if (con->state != CEPH_CON_S_OPEN) {
2421 con->error_msg = "protocol error, unexpected keepalive2_ack";
2425 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2426 ceph_decode_timespec64(&con->last_keepalive_ack, p);
2428 dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2429 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2434 pr_err("failed to decode keepalive2_ack\n");
2438 static int process_ack(struct ceph_connection *con, void *p, void *end)
2442 if (con->state != CEPH_CON_S_OPEN) {
2443 con->error_msg = "protocol error, unexpected ack";
2447 ceph_decode_64_safe(&p, end, seq, bad);
2449 dout("%s con %p seq %llu\n", __func__, con, seq);
2450 ceph_con_discard_sent(con, seq);
2454 pr_err("failed to decode ack\n");
2458 static int process_control(struct ceph_connection *con, void *p, void *end)
2460 int tag = con->v2.in_desc.fd_tag;
2463 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2466 case FRAME_TAG_HELLO:
2467 ret = process_hello(con, p, end);
2469 case FRAME_TAG_AUTH_BAD_METHOD:
2470 ret = process_auth_bad_method(con, p, end);
2472 case FRAME_TAG_AUTH_REPLY_MORE:
2473 ret = process_auth_reply_more(con, p, end);
2475 case FRAME_TAG_AUTH_DONE:
2476 ret = process_auth_done(con, p, end);
2478 case FRAME_TAG_AUTH_SIGNATURE:
2479 ret = process_auth_signature(con, p, end);
2481 case FRAME_TAG_SERVER_IDENT:
2482 ret = process_server_ident(con, p, end);
2484 case FRAME_TAG_IDENT_MISSING_FEATURES:
2485 ret = process_ident_missing_features(con, p, end);
2487 case FRAME_TAG_SESSION_RECONNECT_OK:
2488 ret = process_session_reconnect_ok(con, p, end);
2490 case FRAME_TAG_SESSION_RETRY:
2491 ret = process_session_retry(con, p, end);
2493 case FRAME_TAG_SESSION_RETRY_GLOBAL:
2494 ret = process_session_retry_global(con, p, end);
2496 case FRAME_TAG_SESSION_RESET:
2497 ret = process_session_reset(con, p, end);
2499 case FRAME_TAG_KEEPALIVE2_ACK:
2500 ret = process_keepalive2_ack(con, p, end);
2503 ret = process_ack(con, p, end);
2506 pr_err("bad tag %d\n", tag);
2507 con->error_msg = "protocol error, bad tag";
2511 dout("%s con %p error %d\n", __func__, con, ret);
2515 prepare_read_preamble(con);
2521 * 1 - con->in_msg set, read message
2525 static int process_message_header(struct ceph_connection *con,
2528 struct ceph_frame_desc *desc = &con->v2.in_desc;
2529 struct ceph_msg_header2 *hdr2 = p;
2530 struct ceph_msg_header hdr;
2536 seq = le64_to_cpu(hdr2->seq);
2537 if ((s64)seq - (s64)con->in_seq < 1) {
2538 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2539 ENTITY_NAME(con->peer_name),
2540 ceph_pr_addr(&con->peer_addr),
2541 seq, con->in_seq + 1);
2544 if ((s64)seq - (s64)con->in_seq > 1) {
2545 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2546 con->error_msg = "bad message sequence # for incoming message";
2550 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2552 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2553 desc->fd_lens[3], &con->peer_name);
2554 ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2558 WARN_ON(!con->in_msg ^ skip);
2562 WARN_ON(!con->in_msg);
2563 WARN_ON(con->in_msg->con != con);
2567 static int process_message(struct ceph_connection *con)
2569 ceph_con_process_message(con);
2572 * We could have been closed by ceph_con_close() because
2573 * ceph_con_process_message() temporarily drops con->mutex.
2575 if (con->state != CEPH_CON_S_OPEN) {
2576 dout("%s con %p state changed to %d\n", __func__, con,
2581 prepare_read_preamble(con);
2585 static int __handle_control(struct ceph_connection *con, void *p)
2587 void *end = p + con->v2.in_desc.fd_lens[0];
2588 struct ceph_msg *msg;
2591 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2592 return process_control(con, p, end);
2594 ret = process_message_header(con, p, end);
2598 prepare_skip_message(con);
2602 msg = con->in_msg; /* set in process_message_header() */
2603 if (!front_len(msg) && !middle_len(msg)) {
2605 return process_message(con);
2607 prepare_read_data(con);
2611 reset_in_kvecs(con);
2612 if (front_len(msg)) {
2613 WARN_ON(front_len(msg) > msg->front_alloc_len);
2614 add_in_kvec(con, msg->front.iov_base, front_len(msg));
2615 msg->front.iov_len = front_len(msg);
2617 if (con_secure(con) && need_padding(front_len(msg)))
2618 add_in_kvec(con, FRONT_PAD(con->v2.in_buf),
2619 padding_len(front_len(msg)));
2621 msg->front.iov_len = 0;
2623 if (middle_len(msg)) {
2624 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2625 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
2626 msg->middle->vec.iov_len = middle_len(msg);
2628 if (con_secure(con) && need_padding(middle_len(msg)))
2629 add_in_kvec(con, MIDDLE_PAD(con->v2.in_buf),
2630 padding_len(middle_len(msg)));
2631 } else if (msg->middle) {
2632 msg->middle->vec.iov_len = 0;
2635 if (data_len(msg)) {
2636 con->v2.in_state = IN_S_PREPARE_READ_DATA;
2638 add_in_kvec(con, con->v2.in_buf,
2639 con_secure(con) ? CEPH_EPILOGUE_SECURE_LEN :
2640 CEPH_EPILOGUE_PLAIN_LEN);
2641 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2646 static int handle_preamble(struct ceph_connection *con)
2648 struct ceph_frame_desc *desc = &con->v2.in_desc;
2651 if (con_secure(con)) {
2652 ret = decrypt_preamble(con);
2654 if (ret == -EBADMSG)
2655 con->error_msg = "integrity error, bad preamble auth tag";
2660 ret = decode_preamble(con->v2.in_buf, desc);
2662 if (ret == -EBADMSG)
2663 con->error_msg = "integrity error, bad crc";
2665 con->error_msg = "protocol error, bad preamble";
2669 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2670 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2671 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2673 if (!con_secure(con))
2674 return prepare_read_control(con);
2676 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2677 return prepare_read_control_remainder(con);
2679 return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2682 static int handle_control(struct ceph_connection *con)
2684 int ctrl_len = con->v2.in_desc.fd_lens[0];
2688 WARN_ON(con_secure(con));
2690 ret = verify_control_crc(con);
2692 con->error_msg = "integrity error, bad crc";
2696 if (con->state == CEPH_CON_S_V2_AUTH) {
2697 buf = alloc_conn_buf(con, ctrl_len);
2701 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2702 return __handle_control(con, buf);
2705 return __handle_control(con, con->v2.in_kvecs[0].iov_base);
2708 static int handle_control_remainder(struct ceph_connection *con)
2712 WARN_ON(!con_secure(con));
2714 ret = decrypt_control_remainder(con);
2716 if (ret == -EBADMSG)
2717 con->error_msg = "integrity error, bad control remainder auth tag";
2721 return __handle_control(con, con->v2.in_kvecs[0].iov_base -
2722 CEPH_PREAMBLE_INLINE_LEN);
2725 static int handle_epilogue(struct ceph_connection *con)
2727 u32 front_crc, middle_crc, data_crc;
2730 if (con_secure(con)) {
2731 ret = decrypt_message(con);
2733 if (ret == -EBADMSG)
2734 con->error_msg = "integrity error, bad epilogue auth tag";
2738 /* just late_status */
2739 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
2741 con->error_msg = "protocol error, bad epilogue";
2745 ret = decode_epilogue(con->v2.in_buf, &front_crc,
2746 &middle_crc, &data_crc);
2748 con->error_msg = "protocol error, bad epilogue";
2752 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
2755 con->error_msg = "integrity error, bad crc";
2760 return process_message(con);
2763 static void finish_skip(struct ceph_connection *con)
2765 dout("%s con %p\n", __func__, con);
2767 if (con_secure(con))
2768 gcm_inc_nonce(&con->v2.in_gcm_nonce);
2773 static int populate_in_iter(struct ceph_connection *con)
2777 dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
2779 WARN_ON(iov_iter_count(&con->v2.in_iter));
2781 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
2782 ret = process_banner_prefix(con);
2783 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
2784 ret = process_banner_payload(con);
2785 } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
2786 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
2787 con->state == CEPH_CON_S_OPEN) {
2788 switch (con->v2.in_state) {
2789 case IN_S_HANDLE_PREAMBLE:
2790 ret = handle_preamble(con);
2792 case IN_S_HANDLE_CONTROL:
2793 ret = handle_control(con);
2795 case IN_S_HANDLE_CONTROL_REMAINDER:
2796 ret = handle_control_remainder(con);
2798 case IN_S_PREPARE_READ_DATA:
2799 prepare_read_data(con);
2802 case IN_S_PREPARE_READ_DATA_CONT:
2803 prepare_read_data_cont(con);
2806 case IN_S_HANDLE_EPILOGUE:
2807 ret = handle_epilogue(con);
2809 case IN_S_FINISH_SKIP:
2814 WARN(1, "bad in_state %d", con->v2.in_state);
2818 WARN(1, "bad state %d", con->state);
2822 dout("%s con %p error %d\n", __func__, con, ret);
2826 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2828 dout("%s con %p populated %zu\n", __func__, con,
2829 iov_iter_count(&con->v2.in_iter));
2833 int ceph_con_v2_try_read(struct ceph_connection *con)
2837 dout("%s con %p state %d need %zu\n", __func__, con, con->state,
2838 iov_iter_count(&con->v2.in_iter));
2840 if (con->state == CEPH_CON_S_PREOPEN)
2844 * We should always have something pending here. If not,
2845 * avoid calling populate_in_iter() as if we read something
2846 * (ceph_tcp_recv() would immediately return 1).
2848 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2852 ret = ceph_tcp_recv(con);
2856 ret = populate_in_iter(con);
2858 if (ret && ret != -EAGAIN && !con->error_msg)
2859 con->error_msg = "read processing error";
2865 static void queue_data(struct ceph_connection *con)
2869 con->v2.out_epil.data_crc = -1;
2870 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
2871 data_len(con->out_msg));
2873 get_bvec_at(&con->v2.out_cursor, &bv);
2874 set_out_bvec(con, &bv, true);
2875 con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
2878 static void queue_data_cont(struct ceph_connection *con)
2882 con->v2.out_epil.data_crc = ceph_crc32c_page(
2883 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
2884 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
2886 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
2887 if (con->v2.out_cursor.total_resid) {
2888 get_bvec_at(&con->v2.out_cursor, &bv);
2889 set_out_bvec(con, &bv, true);
2890 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
2895 * We've written all data. Queue epilogue. Once it's written,
2898 reset_out_kvecs(con);
2899 prepare_epilogue_plain(con, false);
2900 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2903 static void queue_enc_page(struct ceph_connection *con)
2907 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
2908 con->v2.out_enc_resid);
2909 WARN_ON(!con->v2.out_enc_resid);
2911 bv.bv_page = con->v2.out_enc_pages[con->v2.out_enc_i];
2913 bv.bv_len = min(con->v2.out_enc_resid, (int)PAGE_SIZE);
2915 set_out_bvec(con, &bv, false);
2916 con->v2.out_enc_i++;
2917 con->v2.out_enc_resid -= bv.bv_len;
2919 if (con->v2.out_enc_resid) {
2920 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
2925 * We've queued the last piece of ciphertext (ending with
2926 * epilogue) + auth tag. Once it's written, we are done.
2928 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
2929 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2932 static void queue_zeros(struct ceph_connection *con)
2934 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
2936 if (con->v2.out_zero) {
2937 set_out_bvec_zero(con);
2938 con->v2.out_zero -= con->v2.out_bvec.bv_len;
2939 con->v2.out_state = OUT_S_QUEUE_ZEROS;
2944 * We've zero-filled everything up to epilogue. Queue epilogue
2945 * with late_status set to ABORTED and crcs adjusted for zeros.
2946 * Once it's written, we are done patching up for the revoke.
2948 reset_out_kvecs(con);
2949 prepare_epilogue_plain(con, true);
2950 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2953 static void finish_message(struct ceph_connection *con)
2955 dout("%s con %p msg %p\n", __func__, con, con->out_msg);
2957 /* we end up here both plain and secure modes */
2958 if (con->v2.out_enc_pages) {
2959 WARN_ON(!con->v2.out_enc_page_cnt);
2960 ceph_release_page_vector(con->v2.out_enc_pages,
2961 con->v2.out_enc_page_cnt);
2962 con->v2.out_enc_pages = NULL;
2963 con->v2.out_enc_page_cnt = 0;
2965 /* message may have been revoked */
2967 ceph_msg_put(con->out_msg);
2968 con->out_msg = NULL;
2971 con->v2.out_state = OUT_S_GET_NEXT;
2974 static int populate_out_iter(struct ceph_connection *con)
2978 dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
2980 WARN_ON(iov_iter_count(&con->v2.out_iter));
2982 if (con->state != CEPH_CON_S_OPEN) {
2983 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
2984 con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
2985 goto nothing_pending;
2988 switch (con->v2.out_state) {
2989 case OUT_S_QUEUE_DATA:
2990 WARN_ON(!con->out_msg);
2993 case OUT_S_QUEUE_DATA_CONT:
2994 WARN_ON(!con->out_msg);
2995 queue_data_cont(con);
2997 case OUT_S_QUEUE_ENC_PAGE:
2998 queue_enc_page(con);
3000 case OUT_S_QUEUE_ZEROS:
3001 WARN_ON(con->out_msg); /* revoked */
3004 case OUT_S_FINISH_MESSAGE:
3005 finish_message(con);
3007 case OUT_S_GET_NEXT:
3010 WARN(1, "bad out_state %d", con->v2.out_state);
3014 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3015 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3016 ret = prepare_keepalive2(con);
3018 pr_err("prepare_keepalive2 failed: %d\n", ret);
3021 } else if (!list_empty(&con->out_queue)) {
3022 ceph_con_get_out_msg(con);
3023 ret = prepare_message(con);
3025 pr_err("prepare_message failed: %d\n", ret);
3028 } else if (con->in_seq > con->in_seq_acked) {
3029 ret = prepare_ack(con);
3031 pr_err("prepare_ack failed: %d\n", ret);
3035 goto nothing_pending;
3039 if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3041 dout("%s con %p populated %zu\n", __func__, con,
3042 iov_iter_count(&con->v2.out_iter));
3046 WARN_ON(iov_iter_count(&con->v2.out_iter));
3047 dout("%s con %p nothing pending\n", __func__, con);
3048 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3052 int ceph_con_v2_try_write(struct ceph_connection *con)
3056 dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3057 iov_iter_count(&con->v2.out_iter));
3059 /* open the socket first? */
3060 if (con->state == CEPH_CON_S_PREOPEN) {
3061 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3064 * Always bump global_seq. Bump connect_seq only if
3065 * there is a session (i.e. we are reconnecting and will
3066 * send session_reconnect instead of client_ident).
3068 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3069 if (con->v2.server_cookie)
3070 con->v2.connect_seq++;
3072 ret = prepare_read_banner_prefix(con);
3074 pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3075 con->error_msg = "connect error";
3079 reset_out_kvecs(con);
3080 ret = prepare_banner(con);
3082 pr_err("prepare_banner failed: %d\n", ret);
3083 con->error_msg = "connect error";
3087 ret = ceph_tcp_connect(con);
3089 pr_err("ceph_tcp_connect failed: %d\n", ret);
3090 con->error_msg = "connect error";
3095 if (!iov_iter_count(&con->v2.out_iter)) {
3096 ret = populate_out_iter(con);
3098 if (ret && ret != -EAGAIN && !con->error_msg)
3099 con->error_msg = "write processing error";
3104 tcp_sock_set_cork(con->sock->sk, true);
3106 ret = ceph_tcp_send(con);
3110 ret = populate_out_iter(con);
3112 if (ret && ret != -EAGAIN && !con->error_msg)
3113 con->error_msg = "write processing error";
3118 tcp_sock_set_cork(con->sock->sk, false);
3122 static u32 crc32c_zeros(u32 crc, int zero_len)
3127 len = min(zero_len, (int)PAGE_SIZE);
3128 crc = crc32c(crc, page_address(ceph_zero_page), len);
3135 static void prepare_zero_front(struct ceph_connection *con, int resid)
3139 WARN_ON(!resid || resid > front_len(con->out_msg));
3140 sent = front_len(con->out_msg) - resid;
3141 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3144 con->v2.out_epil.front_crc =
3145 crc32c(-1, con->out_msg->front.iov_base, sent);
3146 con->v2.out_epil.front_crc =
3147 crc32c_zeros(con->v2.out_epil.front_crc, resid);
3149 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3152 con->v2.out_iter.count -= resid;
3153 out_zero_add(con, resid);
3156 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3160 WARN_ON(!resid || resid > middle_len(con->out_msg));
3161 sent = middle_len(con->out_msg) - resid;
3162 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3165 con->v2.out_epil.middle_crc =
3166 crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3167 con->v2.out_epil.middle_crc =
3168 crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3170 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3173 con->v2.out_iter.count -= resid;
3174 out_zero_add(con, resid);
3177 static void prepare_zero_data(struct ceph_connection *con)
3179 dout("%s con %p\n", __func__, con);
3180 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3181 out_zero_add(con, data_len(con->out_msg));
3184 static void revoke_at_queue_data(struct ceph_connection *con)
3189 WARN_ON(!data_len(con->out_msg));
3190 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3191 resid = iov_iter_count(&con->v2.out_iter);
3193 boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3194 if (resid > boundary) {
3196 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3197 dout("%s con %p was sending head\n", __func__, con);
3198 if (front_len(con->out_msg))
3199 prepare_zero_front(con, front_len(con->out_msg));
3200 if (middle_len(con->out_msg))
3201 prepare_zero_middle(con, middle_len(con->out_msg));
3202 prepare_zero_data(con);
3203 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3204 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3208 boundary = middle_len(con->out_msg);
3209 if (resid > boundary) {
3211 dout("%s con %p was sending front\n", __func__, con);
3212 prepare_zero_front(con, resid);
3213 if (middle_len(con->out_msg))
3214 prepare_zero_middle(con, middle_len(con->out_msg));
3215 prepare_zero_data(con);
3221 dout("%s con %p was sending middle\n", __func__, con);
3222 prepare_zero_middle(con, resid);
3223 prepare_zero_data(con);
3227 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3229 int sent, resid; /* current piece of data */
3231 WARN_ON(!data_len(con->out_msg));
3232 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3233 resid = iov_iter_count(&con->v2.out_iter);
3234 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3235 sent = con->v2.out_bvec.bv_len - resid;
3236 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3239 con->v2.out_epil.data_crc = ceph_crc32c_page(
3240 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3241 con->v2.out_bvec.bv_offset, sent);
3242 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3244 WARN_ON(resid > con->v2.out_cursor.total_resid);
3245 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3246 con->v2.out_cursor.total_resid);
3248 con->v2.out_iter.count -= resid;
3249 out_zero_add(con, con->v2.out_cursor.total_resid);
3253 static void revoke_at_finish_message(struct ceph_connection *con)
3258 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3259 resid = iov_iter_count(&con->v2.out_iter);
3261 if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3262 !data_len(con->out_msg)) {
3263 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3264 dout("%s con %p was sending head (empty message) - noop\n",
3269 boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3270 CEPH_EPILOGUE_PLAIN_LEN;
3271 if (resid > boundary) {
3273 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3274 dout("%s con %p was sending head\n", __func__, con);
3275 if (front_len(con->out_msg))
3276 prepare_zero_front(con, front_len(con->out_msg));
3277 if (middle_len(con->out_msg))
3278 prepare_zero_middle(con, middle_len(con->out_msg));
3279 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3280 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3281 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3285 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3286 if (resid > boundary) {
3288 dout("%s con %p was sending front\n", __func__, con);
3289 prepare_zero_front(con, resid);
3290 if (middle_len(con->out_msg))
3291 prepare_zero_middle(con, middle_len(con->out_msg));
3292 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3297 boundary = CEPH_EPILOGUE_PLAIN_LEN;
3298 if (resid > boundary) {
3300 dout("%s con %p was sending middle\n", __func__, con);
3301 prepare_zero_middle(con, resid);
3302 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3308 dout("%s con %p was sending epilogue - noop\n", __func__, con);
3311 void ceph_con_v2_revoke(struct ceph_connection *con)
3313 WARN_ON(con->v2.out_zero);
3315 if (con_secure(con)) {
3316 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3317 con->v2.out_state != OUT_S_FINISH_MESSAGE);
3318 dout("%s con %p secure - noop\n", __func__, con);
3322 switch (con->v2.out_state) {
3323 case OUT_S_QUEUE_DATA:
3324 revoke_at_queue_data(con);
3326 case OUT_S_QUEUE_DATA_CONT:
3327 revoke_at_queue_data_cont(con);
3329 case OUT_S_FINISH_MESSAGE:
3330 revoke_at_finish_message(con);
3333 WARN(1, "bad out_state %d", con->v2.out_state);
3338 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3340 int remaining; /* data + [data padding] + epilogue */
3343 WARN_ON(!data_len(con->in_msg));
3344 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3345 resid = iov_iter_count(&con->v2.in_iter);
3348 if (con_secure(con))
3349 remaining = padded_len(data_len(con->in_msg)) +
3350 CEPH_EPILOGUE_SECURE_LEN;
3352 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3354 dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3356 con->v2.in_iter.count -= resid;
3357 set_in_skip(con, resid + remaining);
3358 con->v2.in_state = IN_S_FINISH_SKIP;
3361 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3363 int recved, resid; /* current piece of data */
3364 int remaining; /* [data padding] + epilogue */
3366 WARN_ON(!data_len(con->in_msg));
3367 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3368 resid = iov_iter_count(&con->v2.in_iter);
3369 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3370 recved = con->v2.in_bvec.bv_len - resid;
3371 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3374 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3375 WARN_ON(resid > con->v2.in_cursor.total_resid);
3377 if (con_secure(con))
3378 remaining = padding_len(data_len(con->in_msg)) +
3379 CEPH_EPILOGUE_SECURE_LEN;
3381 remaining = CEPH_EPILOGUE_PLAIN_LEN;
3383 dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3384 con->v2.in_cursor.total_resid, remaining);
3385 con->v2.in_iter.count -= resid;
3386 set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3387 con->v2.in_state = IN_S_FINISH_SKIP;
3390 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3394 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3395 resid = iov_iter_count(&con->v2.in_iter);
3398 dout("%s con %p resid %d\n", __func__, con, resid);
3399 con->v2.in_iter.count -= resid;
3400 set_in_skip(con, resid);
3401 con->v2.in_state = IN_S_FINISH_SKIP;
3404 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3406 switch (con->v2.in_state) {
3407 case IN_S_PREPARE_READ_DATA:
3408 revoke_at_prepare_read_data(con);
3410 case IN_S_PREPARE_READ_DATA_CONT:
3411 revoke_at_prepare_read_data_cont(con);
3413 case IN_S_HANDLE_EPILOGUE:
3414 revoke_at_handle_epilogue(con);
3417 WARN(1, "bad in_state %d", con->v2.in_state);
3422 bool ceph_con_v2_opened(struct ceph_connection *con)
3424 return con->v2.peer_global_seq;
3427 void ceph_con_v2_reset_session(struct ceph_connection *con)
3429 con->v2.client_cookie = 0;
3430 con->v2.server_cookie = 0;
3431 con->v2.global_seq = 0;
3432 con->v2.connect_seq = 0;
3433 con->v2.peer_global_seq = 0;
3436 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3438 iov_iter_truncate(&con->v2.in_iter, 0);
3439 iov_iter_truncate(&con->v2.out_iter, 0);
3440 con->v2.out_zero = 0;
3442 clear_in_sign_kvecs(con);
3443 clear_out_sign_kvecs(con);
3444 free_conn_bufs(con);
3446 if (con->v2.out_enc_pages) {
3447 WARN_ON(!con->v2.out_enc_page_cnt);
3448 ceph_release_page_vector(con->v2.out_enc_pages,
3449 con->v2.out_enc_page_cnt);
3450 con->v2.out_enc_pages = NULL;
3451 con->v2.out_enc_page_cnt = 0;
3454 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3455 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3456 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3458 if (con->v2.hmac_tfm) {
3459 crypto_free_shash(con->v2.hmac_tfm);
3460 con->v2.hmac_tfm = NULL;
3462 if (con->v2.gcm_req) {
3463 aead_request_free(con->v2.gcm_req);
3464 con->v2.gcm_req = NULL;
3466 if (con->v2.gcm_tfm) {
3467 crypto_free_aead(con->v2.gcm_tfm);
3468 con->v2.gcm_tfm = NULL;