net/ceph/messenger_v2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Ceph msgr2 protocol implementation
   4  *
   5  * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
   6  */
   7
   8 #include <linux/ceph/ceph_debug.h>
   9
  10 #include <crypto/aead.h>
  11 #include <crypto/hash.h>
  12 #include <crypto/sha2.h>
  13 #include <crypto/utils.h>
  14 #include <linux/bvec.h>
  15 #include <linux/crc32c.h>
  16 #include <linux/net.h>
  17 #include <linux/scatterlist.h>
  18 #include <linux/socket.h>
  19 #include <linux/sched/mm.h>
  20 #include <net/sock.h>
  21 #include <net/tcp.h>
  22
  23 #include <linux/ceph/ceph_features.h>
  24 #include <linux/ceph/decode.h>
  25 #include <linux/ceph/libceph.h>
  26 #include <linux/ceph/messenger.h>
  27
  28 #include "crypto.h"  /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
  29
  30 #define FRAME_TAG_HELLO                 1
  31 #define FRAME_TAG_AUTH_REQUEST          2
  32 #define FRAME_TAG_AUTH_BAD_METHOD       3
  33 #define FRAME_TAG_AUTH_REPLY_MORE       4
  34 #define FRAME_TAG_AUTH_REQUEST_MORE     5
  35 #define FRAME_TAG_AUTH_DONE             6
  36 #define FRAME_TAG_AUTH_SIGNATURE        7
  37 #define FRAME_TAG_CLIENT_IDENT          8
  38 #define FRAME_TAG_SERVER_IDENT          9
  39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
  40 #define FRAME_TAG_SESSION_RECONNECT     11
  41 #define FRAME_TAG_SESSION_RESET         12
  42 #define FRAME_TAG_SESSION_RETRY         13
  43 #define FRAME_TAG_SESSION_RETRY_GLOBAL  14
  44 #define FRAME_TAG_SESSION_RECONNECT_OK  15
  45 #define FRAME_TAG_WAIT                  16
  46 #define FRAME_TAG_MESSAGE               17
  47 #define FRAME_TAG_KEEPALIVE2            18
  48 #define FRAME_TAG_KEEPALIVE2_ACK        19
  49 #define FRAME_TAG_ACK                   20
  50
  51 #define FRAME_LATE_STATUS_ABORTED       0x1
  52 #define FRAME_LATE_STATUS_COMPLETE      0xe
  53 #define FRAME_LATE_STATUS_ABORTED_MASK  0xf
  54
  55 #define IN_S_HANDLE_PREAMBLE                    1
  56 #define IN_S_HANDLE_CONTROL                     2
  57 #define IN_S_HANDLE_CONTROL_REMAINDER           3
  58 #define IN_S_PREPARE_READ_DATA                  4
  59 #define IN_S_PREPARE_READ_DATA_CONT             5
  60 #define IN_S_PREPARE_READ_ENC_PAGE              6
  61 #define IN_S_PREPARE_SPARSE_DATA                7
  62 #define IN_S_PREPARE_SPARSE_DATA_CONT           8
  63 #define IN_S_HANDLE_EPILOGUE                    9
  64 #define IN_S_FINISH_SKIP                        10
  65
  66 #define OUT_S_QUEUE_DATA                1
  67 #define OUT_S_QUEUE_DATA_CONT           2
  68 #define OUT_S_QUEUE_ENC_PAGE            3
  69 #define OUT_S_QUEUE_ZEROS               4
  70 #define OUT_S_FINISH_MESSAGE            5
  71 #define OUT_S_GET_NEXT                  6
  72
  73 #define CTRL_BODY(p)    ((void *)(p) + CEPH_PREAMBLE_LEN)
  74 #define FRONT_PAD(p)    ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
  75 #define MIDDLE_PAD(p)   (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
  76 #define DATA_PAD(p)     (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
  77
  78 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
  79
  80 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
  81 {
  82         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
  83         int ret;
  84
  85         msg.msg_iter = *it;
  86         while (iov_iter_count(it)) {
  87                 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
  88                 if (ret <= 0) {
  89                         if (ret == -EAGAIN)
  90                                 ret = 0;
  91                         return ret;
  92                 }
  93
  94                 iov_iter_advance(it, ret);
  95         }
  96
  97         WARN_ON(msg_data_left(&msg));
  98         return 1;
  99 }
 100
 101 /*
 102  * Read as much as possible.
 103  *
 104  * Return:
 105  *   1 - done, nothing (else) to read
 106  *   0 - socket is empty, need to wait
 107  *  <0 - error
 108  */
 109 static int ceph_tcp_recv(struct ceph_connection *con)
 110 {
 111         int ret;
 112
 113         dout("%s con %p %s %zu\n", __func__, con,
 114              iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
 115              iov_iter_count(&con->v2.in_iter));
 116         ret = do_recvmsg(con->sock, &con->v2.in_iter);
 117         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 118              iov_iter_count(&con->v2.in_iter));
 119         return ret;
 120 }
 121
 122 static int do_sendmsg(struct socket *sock, struct iov_iter *it)
 123 {
 124         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
 125         int ret;
 126
 127         msg.msg_iter = *it;
 128         while (iov_iter_count(it)) {
 129                 ret = sock_sendmsg(sock, &msg);
 130                 if (ret <= 0) {
 131                         if (ret == -EAGAIN)
 132                                 ret = 0;
 133                         return ret;
 134                 }
 135
 136                 iov_iter_advance(it, ret);
 137         }
 138
 139         WARN_ON(msg_data_left(&msg));
 140         return 1;
 141 }
 142
 143 static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
 144 {
 145         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
 146         struct bio_vec bv;
 147         int ret;
 148
 149         if (WARN_ON(!iov_iter_is_bvec(it)))
 150                 return -EINVAL;
 151
 152         while (iov_iter_count(it)) {
 153                 /* iov_iter_iovec() for ITER_BVEC */
 154                 bvec_set_page(&bv, it->bvec->bv_page,
 155                               min(iov_iter_count(it),
 156                                   it->bvec->bv_len - it->iov_offset),
 157                               it->bvec->bv_offset + it->iov_offset);
 158
 159                 /*
 160                  * MSG_SPLICE_PAGES cannot properly handle pages with
 161                  * page_count == 0, we need to fall back to sendmsg if
 162                  * that's the case.
 163                  *
 164                  * Same goes for slab pages: skb_can_coalesce() allows
 165                  * coalescing neighboring slab objects into a single frag
 166                  * which triggers one of hardened usercopy checks.
 167                  */
 168                 if (sendpage_ok(bv.bv_page))
 169                         msg.msg_flags |= MSG_SPLICE_PAGES;
 170                 else
 171                         msg.msg_flags &= ~MSG_SPLICE_PAGES;
 172
 173                 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len);
 174                 ret = sock_sendmsg(sock, &msg);
 175                 if (ret <= 0) {
 176                         if (ret == -EAGAIN)
 177                                 ret = 0;
 178                         return ret;
 179                 }
 180
 181                 iov_iter_advance(it, ret);
 182         }
 183
 184         return 1;
 185 }
 186
 187 /*
 188  * Write as much as possible.  The socket is expected to be corked,
 189  * so we don't bother with MSG_MORE here.
 190  *
 191  * Return:
 192  *   1 - done, nothing (else) to write
 193  *   0 - socket is full, need to wait
 194  *  <0 - error
 195  */
 196 static int ceph_tcp_send(struct ceph_connection *con)
 197 {
 198         int ret;
 199
 200         dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
 201              iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
 202         if (con->v2.out_iter_sendpage)
 203                 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
 204         else
 205                 ret = do_sendmsg(con->sock, &con->v2.out_iter);
 206         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 207              iov_iter_count(&con->v2.out_iter));
 208         return ret;
 209 }
 210
 211 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
 212 {
 213         BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
 214         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
 215
 216         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
 217         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
 218         con->v2.in_kvec_cnt++;
 219
 220         con->v2.in_iter.nr_segs++;
 221         con->v2.in_iter.count += len;
 222 }
 223
 224 static void reset_in_kvecs(struct ceph_connection *con)
 225 {
 226         WARN_ON(iov_iter_count(&con->v2.in_iter));
 227
 228         con->v2.in_kvec_cnt = 0;
 229         iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
 230 }
 231
 232 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
 233 {
 234         WARN_ON(iov_iter_count(&con->v2.in_iter));
 235
 236         con->v2.in_bvec = *bv;
 237         iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
 238 }
 239
 240 static void set_in_skip(struct ceph_connection *con, int len)
 241 {
 242         WARN_ON(iov_iter_count(&con->v2.in_iter));
 243
 244         dout("%s con %p len %d\n", __func__, con, len);
 245         iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
 246 }
 247
 248 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
 249 {
 250         BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
 251         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
 252         WARN_ON(con->v2.out_zero);
 253
 254         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
 255         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
 256         con->v2.out_kvec_cnt++;
 257
 258         con->v2.out_iter.nr_segs++;
 259         con->v2.out_iter.count += len;
 260 }
 261
 262 static void reset_out_kvecs(struct ceph_connection *con)
 263 {
 264         WARN_ON(iov_iter_count(&con->v2.out_iter));
 265         WARN_ON(con->v2.out_zero);
 266
 267         con->v2.out_kvec_cnt = 0;
 268
 269         iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
 270         con->v2.out_iter_sendpage = false;
 271 }
 272
 273 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
 274                          bool zerocopy)
 275 {
 276         WARN_ON(iov_iter_count(&con->v2.out_iter));
 277         WARN_ON(con->v2.out_zero);
 278
 279         con->v2.out_bvec = *bv;
 280         con->v2.out_iter_sendpage = zerocopy;
 281         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 282                       con->v2.out_bvec.bv_len);
 283 }
 284
 285 static void set_out_bvec_zero(struct ceph_connection *con)
 286 {
 287         WARN_ON(iov_iter_count(&con->v2.out_iter));
 288         WARN_ON(!con->v2.out_zero);
 289
 290         bvec_set_page(&con->v2.out_bvec, ceph_zero_page,
 291                       min(con->v2.out_zero, (int)PAGE_SIZE), 0);
 292         con->v2.out_iter_sendpage = true;
 293         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 294                       con->v2.out_bvec.bv_len);
 295 }
 296
 297 static void out_zero_add(struct ceph_connection *con, int len)
 298 {
 299         dout("%s con %p len %d\n", __func__, con, len);
 300         con->v2.out_zero += len;
 301 }
 302
 303 static void *alloc_conn_buf(struct ceph_connection *con, int len)
 304 {
 305         void *buf;
 306
 307         dout("%s con %p len %d\n", __func__, con, len);
 308
 309         if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
 310                 return NULL;
 311
 312         buf = kvmalloc(len, GFP_NOIO);
 313         if (!buf)
 314                 return NULL;
 315
 316         con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
 317         return buf;
 318 }
 319
 320 static void free_conn_bufs(struct ceph_connection *con)
 321 {
 322         while (con->v2.conn_buf_cnt)
 323                 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
 324 }
 325
 326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
 327 {
 328         BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
 329
 330         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
 331         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
 332         con->v2.in_sign_kvec_cnt++;
 333 }
 334
 335 static void clear_in_sign_kvecs(struct ceph_connection *con)
 336 {
 337         con->v2.in_sign_kvec_cnt = 0;
 338 }
 339
 340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
 341 {
 342         BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
 343
 344         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
 345         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
 346         con->v2.out_sign_kvec_cnt++;
 347 }
 348
 349 static void clear_out_sign_kvecs(struct ceph_connection *con)
 350 {
 351         con->v2.out_sign_kvec_cnt = 0;
 352 }
 353
 354 static bool con_secure(struct ceph_connection *con)
 355 {
 356         return con->v2.con_mode == CEPH_CON_MODE_SECURE;
 357 }
 358
 359 static int front_len(const struct ceph_msg *msg)
 360 {
 361         return le32_to_cpu(msg->hdr.front_len);
 362 }
 363
 364 static int middle_len(const struct ceph_msg *msg)
 365 {
 366         return le32_to_cpu(msg->hdr.middle_len);
 367 }
 368
 369 static int data_len(const struct ceph_msg *msg)
 370 {
 371         return le32_to_cpu(msg->hdr.data_len);
 372 }
 373
 374 static bool need_padding(int len)
 375 {
 376         return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
 377 }
 378
 379 static int padded_len(int len)
 380 {
 381         return ALIGN(len, CEPH_GCM_BLOCK_LEN);
 382 }
 383
 384 static int padding_len(int len)
 385 {
 386         return padded_len(len) - len;
 387 }
 388
 389 /* preamble + control segment */
 390 static int head_onwire_len(int ctrl_len, bool secure)
 391 {
 392         int head_len;
 393         int rem_len;
 394
 395         BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
 396
 397         if (secure) {
 398                 head_len = CEPH_PREAMBLE_SECURE_LEN;
 399                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
 400                         rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
 401                         head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
 402                 }
 403         } else {
 404                 head_len = CEPH_PREAMBLE_PLAIN_LEN;
 405                 if (ctrl_len)
 406                         head_len += ctrl_len + CEPH_CRC_LEN;
 407         }
 408         return head_len;
 409 }
 410
 411 /* front, middle and data segments + epilogue */
 412 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
 413                              bool secure)
 414 {
 415         BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
 416                middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
 417                data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
 418
 419         if (!front_len && !middle_len && !data_len)
 420                 return 0;
 421
 422         if (!secure)
 423                 return front_len + middle_len + data_len +
 424                        CEPH_EPILOGUE_PLAIN_LEN;
 425
 426         return padded_len(front_len) + padded_len(middle_len) +
 427                padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
 428 }
 429
 430 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
 431 {
 432         return __tail_onwire_len(front_len(msg), middle_len(msg),
 433                                  data_len(msg), secure);
 434 }
 435
 436 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
 437 #define MESSAGE_HEAD_PLAIN_LEN  (CEPH_PREAMBLE_PLAIN_LEN +              \
 438                                  sizeof(struct ceph_msg_header2) +      \
 439                                  CEPH_CRC_LEN)
 440
 441 static const int frame_aligns[] = {
 442         sizeof(void *),
 443         sizeof(void *),
 444         sizeof(void *),
 445         PAGE_SIZE
 446 };
 447
 448 /*
 449  * Discards trailing empty segments, unless there is just one segment.
 450  * A frame always has at least one (possibly empty) segment.
 451  */
 452 static int calc_segment_count(const int *lens, int len_cnt)
 453 {
 454         int i;
 455
 456         for (i = len_cnt - 1; i >= 0; i--) {
 457                 if (lens[i])
 458                         return i + 1;
 459         }
 460
 461         return 1;
 462 }
 463
 464 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
 465                             const int *lens, int len_cnt)
 466 {
 467         int i;
 468
 469         memset(desc, 0, sizeof(*desc));
 470
 471         desc->fd_tag = tag;
 472         desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
 473         BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
 474         for (i = 0; i < desc->fd_seg_cnt; i++) {
 475                 desc->fd_lens[i] = lens[i];
 476                 desc->fd_aligns[i] = frame_aligns[i];
 477         }
 478 }
 479
 480 /*
 481  * Preamble crc covers everything up to itself (28 bytes) and
 482  * is calculated and verified irrespective of the connection mode
 483  * (i.e. even if the frame is encrypted).
 484  */
 485 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
 486 {
 487         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 488         void *start = p;
 489         int i;
 490
 491         memset(p, 0, CEPH_PREAMBLE_LEN);
 492
 493         ceph_encode_8(&p, desc->fd_tag);
 494         ceph_encode_8(&p, desc->fd_seg_cnt);
 495         for (i = 0; i < desc->fd_seg_cnt; i++) {
 496                 ceph_encode_32(&p, desc->fd_lens[i]);
 497                 ceph_encode_16(&p, desc->fd_aligns[i]);
 498         }
 499
 500         put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
 501 }
 502
 503 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
 504 {
 505         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 506         u32 crc, expected_crc;
 507         int i;
 508
 509         crc = crc32c(0, p, crcp - p);
 510         expected_crc = get_unaligned_le32(crcp);
 511         if (crc != expected_crc) {
 512                 pr_err("bad preamble crc, calculated %u, expected %u\n",
 513                        crc, expected_crc);
 514                 return -EBADMSG;
 515         }
 516
 517         memset(desc, 0, sizeof(*desc));
 518
 519         desc->fd_tag = ceph_decode_8(&p);
 520         desc->fd_seg_cnt = ceph_decode_8(&p);
 521         if (desc->fd_seg_cnt < 1 ||
 522             desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
 523                 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
 524                 return -EINVAL;
 525         }
 526         for (i = 0; i < desc->fd_seg_cnt; i++) {
 527                 desc->fd_lens[i] = ceph_decode_32(&p);
 528                 desc->fd_aligns[i] = ceph_decode_16(&p);
 529         }
 530
 531         if (desc->fd_lens[0] < 0 ||
 532             desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
 533                 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
 534                 return -EINVAL;
 535         }
 536         if (desc->fd_lens[1] < 0 ||
 537             desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
 538                 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
 539                 return -EINVAL;
 540         }
 541         if (desc->fd_lens[2] < 0 ||
 542             desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
 543                 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
 544                 return -EINVAL;
 545         }
 546         if (desc->fd_lens[3] < 0 ||
 547             desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
 548                 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
 549                 return -EINVAL;
 550         }
 551
 552         /*
 553          * This would fire for FRAME_TAG_WAIT (it has one empty
 554          * segment), but we should never get it as client.
 555          */
 556         if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
 557                 pr_err("last segment empty, segment count %d\n",
 558                        desc->fd_seg_cnt);
 559                 return -EINVAL;
 560         }
 561
 562         return 0;
 563 }
 564
 565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
 566 {
 567         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 568                                                  FRAME_LATE_STATUS_COMPLETE;
 569         cpu_to_le32s(&con->v2.out_epil.front_crc);
 570         cpu_to_le32s(&con->v2.out_epil.middle_crc);
 571         cpu_to_le32s(&con->v2.out_epil.data_crc);
 572 }
 573
 574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
 575 {
 576         memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
 577         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 578                                                  FRAME_LATE_STATUS_COMPLETE;
 579 }
 580
 581 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
 582                            u32 *data_crc)
 583 {
 584         u8 late_status;
 585
 586         late_status = ceph_decode_8(&p);
 587         if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
 588                         FRAME_LATE_STATUS_COMPLETE) {
 589                 /* we should never get an aborted message as client */
 590                 pr_err("bad late_status 0x%x\n", late_status);
 591                 return -EINVAL;
 592         }
 593
 594         if (front_crc && middle_crc && data_crc) {
 595                 *front_crc = ceph_decode_32(&p);
 596                 *middle_crc = ceph_decode_32(&p);
 597                 *data_crc = ceph_decode_32(&p);
 598         }
 599
 600         return 0;
 601 }
 602
 603 static void fill_header(struct ceph_msg_header *hdr,
 604                         const struct ceph_msg_header2 *hdr2,
 605                         int front_len, int middle_len, int data_len,
 606                         const struct ceph_entity_name *peer_name)
 607 {
 608         hdr->seq = hdr2->seq;
 609         hdr->tid = hdr2->tid;
 610         hdr->type = hdr2->type;
 611         hdr->priority = hdr2->priority;
 612         hdr->version = hdr2->version;
 613         hdr->front_len = cpu_to_le32(front_len);
 614         hdr->middle_len = cpu_to_le32(middle_len);
 615         hdr->data_len = cpu_to_le32(data_len);
 616         hdr->data_off = hdr2->data_off;
 617         hdr->src = *peer_name;
 618         hdr->compat_version = hdr2->compat_version;
 619         hdr->reserved = 0;
 620         hdr->crc = 0;
 621 }
 622
 623 static void fill_header2(struct ceph_msg_header2 *hdr2,
 624                          const struct ceph_msg_header *hdr, u64 ack_seq)
 625 {
 626         hdr2->seq = hdr->seq;
 627         hdr2->tid = hdr->tid;
 628         hdr2->type = hdr->type;
 629         hdr2->priority = hdr->priority;
 630         hdr2->version = hdr->version;
 631         hdr2->data_pre_padding_len = 0;
 632         hdr2->data_off = hdr->data_off;
 633         hdr2->ack_seq = cpu_to_le64(ack_seq);
 634         hdr2->flags = 0;
 635         hdr2->compat_version = hdr->compat_version;
 636         hdr2->reserved = 0;
 637 }
 638
 639 static int verify_control_crc(struct ceph_connection *con)
 640 {
 641         int ctrl_len = con->v2.in_desc.fd_lens[0];
 642         u32 crc, expected_crc;
 643
 644         WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
 645         WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
 646
 647         crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
 648         expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
 649         if (crc != expected_crc) {
 650                 pr_err("bad control crc, calculated %u, expected %u\n",
 651                        crc, expected_crc);
 652                 return -EBADMSG;
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
 659                                 u32 middle_crc, u32 data_crc)
 660 {
 661         if (front_len(con->in_msg)) {
 662                 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
 663                                            front_len(con->in_msg));
 664         } else {
 665                 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
 666                 con->in_front_crc = -1;
 667         }
 668
 669         if (middle_len(con->in_msg))
 670                 con->in_middle_crc = crc32c(-1,
 671                                             con->in_msg->middle->vec.iov_base,
 672                                             middle_len(con->in_msg));
 673         else if (data_len(con->in_msg))
 674                 con->in_middle_crc = -1;
 675         else
 676                 con->in_middle_crc = 0;
 677
 678         if (!data_len(con->in_msg))
 679                 con->in_data_crc = 0;
 680
 681         dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
 682              con->in_front_crc, con->in_middle_crc, con->in_data_crc);
 683
 684         if (con->in_front_crc != front_crc) {
 685                 pr_err("bad front crc, calculated %u, expected %u\n",
 686                        con->in_front_crc, front_crc);
 687                 return -EBADMSG;
 688         }
 689         if (con->in_middle_crc != middle_crc) {
 690                 pr_err("bad middle crc, calculated %u, expected %u\n",
 691                        con->in_middle_crc, middle_crc);
 692                 return -EBADMSG;
 693         }
 694         if (con->in_data_crc != data_crc) {
 695                 pr_err("bad data crc, calculated %u, expected %u\n",
 696                        con->in_data_crc, data_crc);
 697                 return -EBADMSG;
 698         }
 699
 700         return 0;
 701 }
 702
 703 static int setup_crypto(struct ceph_connection *con,
 704                         const u8 *session_key, int session_key_len,
 705                         const u8 *con_secret, int con_secret_len)
 706 {
 707         unsigned int noio_flag;
 708         int ret;
 709
 710         dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
 711              __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
 712         WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
 713
 714         if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
 715             con->v2.con_mode != CEPH_CON_MODE_SECURE) {
 716                 pr_err("bad con_mode %d\n", con->v2.con_mode);
 717                 return -EINVAL;
 718         }
 719
 720         if (!session_key_len) {
 721                 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
 722                 WARN_ON(con_secret_len);
 723                 return 0;  /* auth_none */
 724         }
 725
 726         noio_flag = memalloc_noio_save();
 727         con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
 728         memalloc_noio_restore(noio_flag);
 729         if (IS_ERR(con->v2.hmac_tfm)) {
 730                 ret = PTR_ERR(con->v2.hmac_tfm);
 731                 con->v2.hmac_tfm = NULL;
 732                 pr_err("failed to allocate hmac tfm context: %d\n", ret);
 733                 return ret;
 734         }
 735
 736         ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
 737                                   session_key_len);
 738         if (ret) {
 739                 pr_err("failed to set hmac key: %d\n", ret);
 740                 return ret;
 741         }
 742
 743         if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
 744                 WARN_ON(con_secret_len);
 745                 return 0;  /* auth_x, plain mode */
 746         }
 747
 748         if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
 749                 pr_err("con_secret too small %d\n", con_secret_len);
 750                 return -EINVAL;
 751         }
 752
 753         noio_flag = memalloc_noio_save();
 754         con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
 755         memalloc_noio_restore(noio_flag);
 756         if (IS_ERR(con->v2.gcm_tfm)) {
 757                 ret = PTR_ERR(con->v2.gcm_tfm);
 758                 con->v2.gcm_tfm = NULL;
 759                 pr_err("failed to allocate gcm tfm context: %d\n", ret);
 760                 return ret;
 761         }
 762
 763         WARN_ON((unsigned long)con_secret &
 764                 crypto_aead_alignmask(con->v2.gcm_tfm));
 765         ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
 766         if (ret) {
 767                 pr_err("failed to set gcm key: %d\n", ret);
 768                 return ret;
 769         }
 770
 771         WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
 772         ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
 773         if (ret) {
 774                 pr_err("failed to set gcm tag size: %d\n", ret);
 775                 return ret;
 776         }
 777
 778         con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
 779         if (!con->v2.gcm_req) {
 780                 pr_err("failed to allocate gcm request\n");
 781                 return -ENOMEM;
 782         }
 783
 784         crypto_init_wait(&con->v2.gcm_wait);
 785         aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 786                                   crypto_req_done, &con->v2.gcm_wait);
 787
 788         memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
 789                CEPH_GCM_IV_LEN);
 790         memcpy(&con->v2.out_gcm_nonce,
 791                con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
 792                CEPH_GCM_IV_LEN);
 793         return 0;  /* auth_x, secure mode */
 794 }
 795
 796 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
 797                        int kvec_cnt, u8 *hmac)
 798 {
 799         SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm);  /* tfm arg is ignored */
 800         int ret;
 801         int i;
 802
 803         dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
 804              con->v2.hmac_tfm, kvec_cnt);
 805
 806         if (!con->v2.hmac_tfm) {
 807                 memset(hmac, 0, SHA256_DIGEST_SIZE);
 808                 return 0;  /* auth_none */
 809         }
 810
 811         desc->tfm = con->v2.hmac_tfm;
 812         ret = crypto_shash_init(desc);
 813         if (ret)
 814                 goto out;
 815
 816         for (i = 0; i < kvec_cnt; i++) {
 817                 ret = crypto_shash_update(desc, kvecs[i].iov_base,
 818                                           kvecs[i].iov_len);
 819                 if (ret)
 820                         goto out;
 821         }
 822
 823         ret = crypto_shash_final(desc, hmac);
 824
 825 out:
 826         shash_desc_zero(desc);
 827         return ret;  /* auth_x, both plain and secure modes */
 828 }
 829
 830 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
 831 {
 832         u64 counter;
 833
 834         counter = le64_to_cpu(nonce->counter);
 835         nonce->counter = cpu_to_le64(counter + 1);
 836 }
 837
 838 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
 839                      struct scatterlist *src, struct scatterlist *dst,
 840                      int src_len)
 841 {
 842         struct ceph_gcm_nonce *nonce;
 843         int ret;
 844
 845         nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
 846
 847         aead_request_set_ad(con->v2.gcm_req, 0);  /* no AAD */
 848         aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
 849         ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
 850                                         crypto_aead_decrypt(con->v2.gcm_req),
 851                               &con->v2.gcm_wait);
 852         if (ret)
 853                 return ret;
 854
 855         gcm_inc_nonce(nonce);
 856         return 0;
 857 }
 858
 859 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
 860                         struct bio_vec *bv)
 861 {
 862         struct page *page;
 863         size_t off, len;
 864
 865         WARN_ON(!cursor->total_resid);
 866
 867         /* skip zero-length data items */
 868         while (!cursor->resid)
 869                 ceph_msg_data_advance(cursor, 0);
 870
 871         /* get a piece of data, cursor isn't advanced */
 872         page = ceph_msg_data_next(cursor, &off, &len);
 873         bvec_set_page(bv, page, len, off);
 874 }
 875
 876 static int calc_sg_cnt(void *buf, int buf_len)
 877 {
 878         int sg_cnt;
 879
 880         if (!buf_len)
 881                 return 0;
 882
 883         sg_cnt = need_padding(buf_len) ? 1 : 0;
 884         if (is_vmalloc_addr(buf)) {
 885                 WARN_ON(offset_in_page(buf));
 886                 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
 887         } else {
 888                 sg_cnt++;
 889         }
 890
 891         return sg_cnt;
 892 }
 893
 894 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
 895 {
 896         int data_len = cursor->total_resid;
 897         struct bio_vec bv;
 898         int sg_cnt;
 899
 900         if (!data_len)
 901                 return 0;
 902
 903         sg_cnt = need_padding(data_len) ? 1 : 0;
 904         do {
 905                 get_bvec_at(cursor, &bv);
 906                 sg_cnt++;
 907
 908                 ceph_msg_data_advance(cursor, bv.bv_len);
 909         } while (cursor->total_resid);
 910
 911         return sg_cnt;
 912 }
 913
 914 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
 915 {
 916         void *end = buf + buf_len;
 917         struct page *page;
 918         int len;
 919         void *p;
 920
 921         if (!buf_len)
 922                 return;
 923
 924         if (is_vmalloc_addr(buf)) {
 925                 p = buf;
 926                 do {
 927                         page = vmalloc_to_page(p);
 928                         len = min_t(int, end - p, PAGE_SIZE);
 929                         WARN_ON(!page || !len || offset_in_page(p));
 930                         sg_set_page(*sg, page, len, 0);
 931                         *sg = sg_next(*sg);
 932                         p += len;
 933                 } while (p != end);
 934         } else {
 935                 sg_set_buf(*sg, buf, buf_len);
 936                 *sg = sg_next(*sg);
 937         }
 938
 939         if (need_padding(buf_len)) {
 940                 sg_set_buf(*sg, pad, padding_len(buf_len));
 941                 *sg = sg_next(*sg);
 942         }
 943 }
 944
 945 static void init_sgs_cursor(struct scatterlist **sg,
 946                             struct ceph_msg_data_cursor *cursor, u8 *pad)
 947 {
 948         int data_len = cursor->total_resid;
 949         struct bio_vec bv;
 950
 951         if (!data_len)
 952                 return;
 953
 954         do {
 955                 get_bvec_at(cursor, &bv);
 956                 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
 957                 *sg = sg_next(*sg);
 958
 959                 ceph_msg_data_advance(cursor, bv.bv_len);
 960         } while (cursor->total_resid);
 961
 962         if (need_padding(data_len)) {
 963                 sg_set_buf(*sg, pad, padding_len(data_len));
 964                 *sg = sg_next(*sg);
 965         }
 966 }
 967
 968 /**
 969  * init_sgs_pages: set up scatterlist on an array of page pointers
 970  * @sg:         scatterlist to populate
 971  * @pages:      pointer to page array
 972  * @dpos:       position in the array to start (bytes)
 973  * @dlen:       len to add to sg (bytes)
 974  * @pad:        pointer to pad destination (if any)
 975  *
 976  * Populate the scatterlist from the page array, starting at an arbitrary
 977  * byte in the array and running for a specified length.
 978  */
 979 static void init_sgs_pages(struct scatterlist **sg, struct page **pages,
 980                            int dpos, int dlen, u8 *pad)
 981 {
 982         int idx = dpos >> PAGE_SHIFT;
 983         int off = offset_in_page(dpos);
 984         int resid = dlen;
 985
 986         do {
 987                 int len = min(resid, (int)PAGE_SIZE - off);
 988
 989                 sg_set_page(*sg, pages[idx], len, off);
 990                 *sg = sg_next(*sg);
 991                 off = 0;
 992                 ++idx;
 993                 resid -= len;
 994         } while (resid);
 995
 996         if (need_padding(dlen)) {
 997                 sg_set_buf(*sg, pad, padding_len(dlen));
 998                 *sg = sg_next(*sg);
 999         }
1000 }
1001
1002 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
1003                              u8 *front_pad, u8 *middle_pad, u8 *data_pad,
1004                              void *epilogue, struct page **pages, int dpos,
1005                              bool add_tag)
1006 {
1007         struct ceph_msg_data_cursor cursor;
1008         struct scatterlist *cur_sg;
1009         int dlen = data_len(msg);
1010         int sg_cnt;
1011         int ret;
1012
1013         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
1014                 return 0;
1015
1016         sg_cnt = 1;  /* epilogue + [auth tag] */
1017         if (front_len(msg))
1018                 sg_cnt += calc_sg_cnt(msg->front.iov_base,
1019                                       front_len(msg));
1020         if (middle_len(msg))
1021                 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
1022                                       middle_len(msg));
1023         if (dlen) {
1024                 if (pages) {
1025                         sg_cnt += calc_pages_for(dpos, dlen);
1026                         if (need_padding(dlen))
1027                                 sg_cnt++;
1028                 } else {
1029                         ceph_msg_data_cursor_init(&cursor, msg, dlen);
1030                         sg_cnt += calc_sg_cnt_cursor(&cursor);
1031                 }
1032         }
1033
1034         ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1035         if (ret)
1036                 return ret;
1037
1038         cur_sg = sgt->sgl;
1039         if (front_len(msg))
1040                 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1041                          front_pad);
1042         if (middle_len(msg))
1043                 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1044                          middle_pad);
1045         if (dlen) {
1046                 if (pages) {
1047                         init_sgs_pages(&cur_sg, pages, dpos, dlen, data_pad);
1048                 } else {
1049                         ceph_msg_data_cursor_init(&cursor, msg, dlen);
1050                         init_sgs_cursor(&cur_sg, &cursor, data_pad);
1051                 }
1052         }
1053
1054         WARN_ON(!sg_is_last(cur_sg));
1055         sg_set_buf(cur_sg, epilogue,
1056                    CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1057         return 0;
1058 }
1059
1060 static int decrypt_preamble(struct ceph_connection *con)
1061 {
1062         struct scatterlist sg;
1063
1064         sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1065         return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1066 }
1067
1068 static int decrypt_control_remainder(struct ceph_connection *con)
1069 {
1070         int ctrl_len = con->v2.in_desc.fd_lens[0];
1071         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1072         int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1073         struct scatterlist sgs[2];
1074
1075         WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1076         WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1077
1078         sg_init_table(sgs, 2);
1079         sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1080         sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1081
1082         return gcm_crypt(con, false, sgs, sgs,
1083                          padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1084 }
1085
1086 /* Process sparse read data that lives in a buffer */
1087 static int process_v2_sparse_read(struct ceph_connection *con,
1088                                   struct page **pages, int spos)
1089 {
1090         struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1091         int ret;
1092
1093         for (;;) {
1094                 char *buf = NULL;
1095
1096                 ret = con->ops->sparse_read(con, cursor, &buf);
1097                 if (ret <= 0)
1098                         return ret;
1099
1100                 dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf);
1101
1102                 do {
1103                         int idx = spos >> PAGE_SHIFT;
1104                         int soff = offset_in_page(spos);
1105                         struct page *spage = con->v2.in_enc_pages[idx];
1106                         int len = min_t(int, ret, PAGE_SIZE - soff);
1107
1108                         if (buf) {
1109                                 memcpy_from_page(buf, spage, soff, len);
1110                                 buf += len;
1111                         } else {
1112                                 struct bio_vec bv;
1113
1114                                 get_bvec_at(cursor, &bv);
1115                                 len = min_t(int, len, bv.bv_len);
1116                                 memcpy_page(bv.bv_page, bv.bv_offset,
1117                                             spage, soff, len);
1118                                 ceph_msg_data_advance(cursor, len);
1119                         }
1120                         spos += len;
1121                         ret -= len;
1122                 } while (ret);
1123         }
1124 }
1125
1126 static int decrypt_tail(struct ceph_connection *con)
1127 {
1128         struct sg_table enc_sgt = {};
1129         struct sg_table sgt = {};
1130         struct page **pages = NULL;
1131         bool sparse = con->in_msg->sparse_read;
1132         int dpos = 0;
1133         int tail_len;
1134         int ret;
1135
1136         tail_len = tail_onwire_len(con->in_msg, true);
1137         ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
1138                                         con->v2.in_enc_page_cnt, 0, tail_len,
1139                                         GFP_NOIO);
1140         if (ret)
1141                 goto out;
1142
1143         if (sparse) {
1144                 dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg)));
1145                 pages = con->v2.in_enc_pages;
1146         }
1147
1148         ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1149                                 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1150                                 con->v2.in_buf, pages, dpos, true);
1151         if (ret)
1152                 goto out;
1153
1154         dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1155              con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1156         ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1157         if (ret)
1158                 goto out;
1159
1160         if (sparse && data_len(con->in_msg)) {
1161                 ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos);
1162                 if (ret)
1163                         goto out;
1164         }
1165
1166         WARN_ON(!con->v2.in_enc_page_cnt);
1167         ceph_release_page_vector(con->v2.in_enc_pages,
1168                                  con->v2.in_enc_page_cnt);
1169         con->v2.in_enc_pages = NULL;
1170         con->v2.in_enc_page_cnt = 0;
1171
1172 out:
1173         sg_free_table(&sgt);
1174         sg_free_table(&enc_sgt);
1175         return ret;
1176 }
1177
1178 static int prepare_banner(struct ceph_connection *con)
1179 {
1180         int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1181         void *buf, *p;
1182
1183         buf = alloc_conn_buf(con, buf_len);
1184         if (!buf)
1185                 return -ENOMEM;
1186
1187         p = buf;
1188         ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1189         ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1190         ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1191         ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1192         WARN_ON(p != buf + buf_len);
1193
1194         add_out_kvec(con, buf, buf_len);
1195         add_out_sign_kvec(con, buf, buf_len);
1196         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1197         return 0;
1198 }
1199
1200 /*
1201  * base:
1202  *   preamble
1203  *   control body (ctrl_len bytes)
1204  *   space for control crc
1205  *
1206  * extdata (optional):
1207  *   control body (extdata_len bytes)
1208  *
1209  * Compute control crc and gather base and extdata into:
1210  *
1211  *   preamble
1212  *   control body (ctrl_len + extdata_len bytes)
1213  *   control crc
1214  *
1215  * Preamble should already be encoded at the start of base.
1216  */
1217 static void prepare_head_plain(struct ceph_connection *con, void *base,
1218                                int ctrl_len, void *extdata, int extdata_len,
1219                                bool to_be_signed)
1220 {
1221         int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1222         void *crcp = base + base_len - CEPH_CRC_LEN;
1223         u32 crc;
1224
1225         crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1226         if (extdata_len)
1227                 crc = crc32c(crc, extdata, extdata_len);
1228         put_unaligned_le32(crc, crcp);
1229
1230         if (!extdata_len) {
1231                 add_out_kvec(con, base, base_len);
1232                 if (to_be_signed)
1233                         add_out_sign_kvec(con, base, base_len);
1234                 return;
1235         }
1236
1237         add_out_kvec(con, base, crcp - base);
1238         add_out_kvec(con, extdata, extdata_len);
1239         add_out_kvec(con, crcp, CEPH_CRC_LEN);
1240         if (to_be_signed) {
1241                 add_out_sign_kvec(con, base, crcp - base);
1242                 add_out_sign_kvec(con, extdata, extdata_len);
1243                 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1244         }
1245 }
1246
1247 static int prepare_head_secure_small(struct ceph_connection *con,
1248                                      void *base, int ctrl_len)
1249 {
1250         struct scatterlist sg;
1251         int ret;
1252
1253         /* inline buffer padding? */
1254         if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1255                 memset(CTRL_BODY(base) + ctrl_len, 0,
1256                        CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1257
1258         sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1259         ret = gcm_crypt(con, true, &sg, &sg,
1260                         CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1261         if (ret)
1262                 return ret;
1263
1264         add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1265         return 0;
1266 }
1267
1268 /*
1269  * base:
1270  *   preamble
1271  *   control body (ctrl_len bytes)
1272  *   space for padding, if needed
1273  *   space for control remainder auth tag
1274  *   space for preamble auth tag
1275  *
1276  * Encrypt preamble and the inline portion, then encrypt the remainder
1277  * and gather into:
1278  *
1279  *   preamble
1280  *   control body (48 bytes)
1281  *   preamble auth tag
1282  *   control body (ctrl_len - 48 bytes)
1283  *   zero padding, if needed
1284  *   control remainder auth tag
1285  *
1286  * Preamble should already be encoded at the start of base.
1287  */
1288 static int prepare_head_secure_big(struct ceph_connection *con,
1289                                    void *base, int ctrl_len)
1290 {
1291         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1292         void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1293         void *rem_tag = rem + padded_len(rem_len);
1294         void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1295         struct scatterlist sgs[2];
1296         int ret;
1297
1298         sg_init_table(sgs, 2);
1299         sg_set_buf(&sgs[0], base, rem - base);
1300         sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1301         ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1302         if (ret)
1303                 return ret;
1304
1305         /* control remainder padding? */
1306         if (need_padding(rem_len))
1307                 memset(rem + rem_len, 0, padding_len(rem_len));
1308
1309         sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1310         ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1311         if (ret)
1312                 return ret;
1313
1314         add_out_kvec(con, base, rem - base);
1315         add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1316         add_out_kvec(con, rem, pmbl_tag - rem);
1317         return 0;
1318 }
1319
1320 static int __prepare_control(struct ceph_connection *con, int tag,
1321                              void *base, int ctrl_len, void *extdata,
1322                              int extdata_len, bool to_be_signed)
1323 {
1324         int total_len = ctrl_len + extdata_len;
1325         struct ceph_frame_desc desc;
1326         int ret;
1327
1328         dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1329              total_len, ctrl_len, extdata_len);
1330
1331         /* extdata may be vmalloc'ed but not base */
1332         if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1333                 return -EINVAL;
1334
1335         init_frame_desc(&desc, tag, &total_len, 1);
1336         encode_preamble(&desc, base);
1337
1338         if (con_secure(con)) {
1339                 if (WARN_ON(extdata_len || to_be_signed))
1340                         return -EINVAL;
1341
1342                 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1343                         /* fully inlined, inline buffer may need padding */
1344                         ret = prepare_head_secure_small(con, base, ctrl_len);
1345                 else
1346                         /* partially inlined, inline buffer is full */
1347                         ret = prepare_head_secure_big(con, base, ctrl_len);
1348                 if (ret)
1349                         return ret;
1350         } else {
1351                 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1352                                    to_be_signed);
1353         }
1354
1355         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1356         return 0;
1357 }
1358
1359 static int prepare_control(struct ceph_connection *con, int tag,
1360                            void *base, int ctrl_len)
1361 {
1362         return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1363 }
1364
1365 static int prepare_hello(struct ceph_connection *con)
1366 {
1367         void *buf, *p;
1368         int ctrl_len;
1369
1370         ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1371         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1372         if (!buf)
1373                 return -ENOMEM;
1374
1375         p = CTRL_BODY(buf);
1376         ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1377         ceph_encode_entity_addr(&p, &con->peer_addr);
1378         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1379
1380         return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1381                                  NULL, 0, true);
1382 }
1383
1384 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1385 #define AUTH_BUF_LEN    (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1386
1387 static int prepare_auth_request(struct ceph_connection *con)
1388 {
1389         void *authorizer, *authorizer_copy;
1390         int ctrl_len, authorizer_len;
1391         void *buf;
1392         int ret;
1393
1394         ctrl_len = AUTH_BUF_LEN;
1395         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1396         if (!buf)
1397                 return -ENOMEM;
1398
1399         mutex_unlock(&con->mutex);
1400         ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1401                                          &authorizer, &authorizer_len);
1402         mutex_lock(&con->mutex);
1403         if (con->state != CEPH_CON_S_V2_HELLO) {
1404                 dout("%s con %p state changed to %d\n", __func__, con,
1405                      con->state);
1406                 return -EAGAIN;
1407         }
1408
1409         dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1410         if (ret)
1411                 return ret;
1412
1413         authorizer_copy = alloc_conn_buf(con, authorizer_len);
1414         if (!authorizer_copy)
1415                 return -ENOMEM;
1416
1417         memcpy(authorizer_copy, authorizer, authorizer_len);
1418
1419         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1420                                  authorizer_copy, authorizer_len, true);
1421 }
1422
1423 static int prepare_auth_request_more(struct ceph_connection *con,
1424                                      void *reply, int reply_len)
1425 {
1426         int ctrl_len, authorizer_len;
1427         void *authorizer;
1428         void *buf;
1429         int ret;
1430
1431         ctrl_len = AUTH_BUF_LEN;
1432         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1433         if (!buf)
1434                 return -ENOMEM;
1435
1436         mutex_unlock(&con->mutex);
1437         ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1438                                                CTRL_BODY(buf), &ctrl_len,
1439                                                &authorizer, &authorizer_len);
1440         mutex_lock(&con->mutex);
1441         if (con->state != CEPH_CON_S_V2_AUTH) {
1442                 dout("%s con %p state changed to %d\n", __func__, con,
1443                      con->state);
1444                 return -EAGAIN;
1445         }
1446
1447         dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1448         if (ret)
1449                 return ret;
1450
1451         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1452                                  ctrl_len, authorizer, authorizer_len, true);
1453 }
1454
1455 static int prepare_auth_signature(struct ceph_connection *con)
1456 {
1457         void *buf;
1458         int ret;
1459
1460         buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1461                                                   con_secure(con)));
1462         if (!buf)
1463                 return -ENOMEM;
1464
1465         ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1466                           CTRL_BODY(buf));
1467         if (ret)
1468                 return ret;
1469
1470         return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1471                                SHA256_DIGEST_SIZE);
1472 }
1473
1474 static int prepare_client_ident(struct ceph_connection *con)
1475 {
1476         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1477         struct ceph_client *client = from_msgr(con->msgr);
1478         u64 global_id = ceph_client_gid(client);
1479         void *buf, *p;
1480         int ctrl_len;
1481
1482         WARN_ON(con->v2.server_cookie);
1483         WARN_ON(con->v2.connect_seq);
1484         WARN_ON(con->v2.peer_global_seq);
1485
1486         if (!con->v2.client_cookie) {
1487                 do {
1488                         get_random_bytes(&con->v2.client_cookie,
1489                                          sizeof(con->v2.client_cookie));
1490                 } while (!con->v2.client_cookie);
1491                 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1492                      con->v2.client_cookie);
1493         } else {
1494                 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1495                      con->v2.client_cookie);
1496         }
1497
1498         dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1499              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1500              ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1501              global_id, con->v2.global_seq, client->supported_features,
1502              client->required_features, con->v2.client_cookie);
1503
1504         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1505                    ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1506         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1507         if (!buf)
1508                 return -ENOMEM;
1509
1510         p = CTRL_BODY(buf);
1511         ceph_encode_8(&p, 2);  /* addrvec marker */
1512         ceph_encode_32(&p, 1);  /* addr_cnt */
1513         ceph_encode_entity_addr(&p, my_addr);
1514         ceph_encode_entity_addr(&p, &con->peer_addr);
1515         ceph_encode_64(&p, global_id);
1516         ceph_encode_64(&p, con->v2.global_seq);
1517         ceph_encode_64(&p, client->supported_features);
1518         ceph_encode_64(&p, client->required_features);
1519         ceph_encode_64(&p, 0);  /* flags */
1520         ceph_encode_64(&p, con->v2.client_cookie);
1521         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1522
1523         return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1524 }
1525
1526 static int prepare_session_reconnect(struct ceph_connection *con)
1527 {
1528         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1529         void *buf, *p;
1530         int ctrl_len;
1531
1532         WARN_ON(!con->v2.client_cookie);
1533         WARN_ON(!con->v2.server_cookie);
1534         WARN_ON(!con->v2.connect_seq);
1535         WARN_ON(!con->v2.peer_global_seq);
1536
1537         dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1538              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1539              con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1540              con->v2.connect_seq, con->in_seq);
1541
1542         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1543         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1544         if (!buf)
1545                 return -ENOMEM;
1546
1547         p = CTRL_BODY(buf);
1548         ceph_encode_8(&p, 2);  /* entity_addrvec_t marker */
1549         ceph_encode_32(&p, 1);  /* my_addrs len */
1550         ceph_encode_entity_addr(&p, my_addr);
1551         ceph_encode_64(&p, con->v2.client_cookie);
1552         ceph_encode_64(&p, con->v2.server_cookie);
1553         ceph_encode_64(&p, con->v2.global_seq);
1554         ceph_encode_64(&p, con->v2.connect_seq);
1555         ceph_encode_64(&p, con->in_seq);
1556         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1557
1558         return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1559 }
1560
1561 static int prepare_keepalive2(struct ceph_connection *con)
1562 {
1563         struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1564         struct timespec64 now;
1565
1566         ktime_get_real_ts64(&now);
1567         dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1568              now.tv_nsec);
1569
1570         ceph_encode_timespec64(ts, &now);
1571
1572         reset_out_kvecs(con);
1573         return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1574                                sizeof(struct ceph_timespec));
1575 }
1576
1577 static int prepare_ack(struct ceph_connection *con)
1578 {
1579         void *p;
1580
1581         dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1582              con->in_seq_acked, con->in_seq);
1583         con->in_seq_acked = con->in_seq;
1584
1585         p = CTRL_BODY(con->v2.out_buf);
1586         ceph_encode_64(&p, con->in_seq_acked);
1587
1588         reset_out_kvecs(con);
1589         return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1590 }
1591
1592 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1593 {
1594         dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1595              con->out_msg, aborted, con->v2.out_epil.front_crc,
1596              con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1597
1598         encode_epilogue_plain(con, aborted);
1599         add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1600 }
1601
1602 /*
1603  * For "used" empty segments, crc is -1.  For unused (trailing)
1604  * segments, crc is 0.
1605  */
1606 static void prepare_message_plain(struct ceph_connection *con)
1607 {
1608         struct ceph_msg *msg = con->out_msg;
1609
1610         prepare_head_plain(con, con->v2.out_buf,
1611                            sizeof(struct ceph_msg_header2), NULL, 0, false);
1612
1613         if (!front_len(msg) && !middle_len(msg)) {
1614                 if (!data_len(msg)) {
1615                         /*
1616                          * Empty message: once the head is written,
1617                          * we are done -- there is no epilogue.
1618                          */
1619                         con->v2.out_state = OUT_S_FINISH_MESSAGE;
1620                         return;
1621                 }
1622
1623                 con->v2.out_epil.front_crc = -1;
1624                 con->v2.out_epil.middle_crc = -1;
1625                 con->v2.out_state = OUT_S_QUEUE_DATA;
1626                 return;
1627         }
1628
1629         if (front_len(msg)) {
1630                 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1631                                                     front_len(msg));
1632                 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1633         } else {
1634                 /* middle (at least) is there, checked above */
1635                 con->v2.out_epil.front_crc = -1;
1636         }
1637
1638         if (middle_len(msg)) {
1639                 con->v2.out_epil.middle_crc =
1640                         crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1641                 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1642         } else {
1643                 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1644         }
1645
1646         if (data_len(msg)) {
1647                 con->v2.out_state = OUT_S_QUEUE_DATA;
1648         } else {
1649                 con->v2.out_epil.data_crc = 0;
1650                 prepare_epilogue_plain(con, false);
1651                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1652         }
1653 }
1654
1655 /*
1656  * Unfortunately the kernel crypto API doesn't support streaming
1657  * (piecewise) operation for AEAD algorithms, so we can't get away
1658  * with a fixed size buffer and a couple sgs.  Instead, we have to
1659  * allocate pages for the entire tail of the message (currently up
1660  * to ~32M) and two sgs arrays (up to ~256K each)...
1661  */
1662 static int prepare_message_secure(struct ceph_connection *con)
1663 {
1664         void *zerop = page_address(ceph_zero_page);
1665         struct sg_table enc_sgt = {};
1666         struct sg_table sgt = {};
1667         struct page **enc_pages;
1668         int enc_page_cnt;
1669         int tail_len;
1670         int ret;
1671
1672         ret = prepare_head_secure_small(con, con->v2.out_buf,
1673                                         sizeof(struct ceph_msg_header2));
1674         if (ret)
1675                 return ret;
1676
1677         tail_len = tail_onwire_len(con->out_msg, true);
1678         if (!tail_len) {
1679                 /*
1680                  * Empty message: once the head is written,
1681                  * we are done -- there is no epilogue.
1682                  */
1683                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1684                 return 0;
1685         }
1686
1687         encode_epilogue_secure(con, false);
1688         ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1689                                 &con->v2.out_epil, NULL, 0, false);
1690         if (ret)
1691                 goto out;
1692
1693         enc_page_cnt = calc_pages_for(0, tail_len);
1694         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1695         if (IS_ERR(enc_pages)) {
1696                 ret = PTR_ERR(enc_pages);
1697                 goto out;
1698         }
1699
1700         WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1701         con->v2.out_enc_pages = enc_pages;
1702         con->v2.out_enc_page_cnt = enc_page_cnt;
1703         con->v2.out_enc_resid = tail_len;
1704         con->v2.out_enc_i = 0;
1705
1706         ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1707                                         0, tail_len, GFP_NOIO);
1708         if (ret)
1709                 goto out;
1710
1711         ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1712                         tail_len - CEPH_GCM_TAG_LEN);
1713         if (ret)
1714                 goto out;
1715
1716         dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1717              con->out_msg, sgt.orig_nents, enc_page_cnt);
1718         con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1719
1720 out:
1721         sg_free_table(&sgt);
1722         sg_free_table(&enc_sgt);
1723         return ret;
1724 }
1725
1726 static int prepare_message(struct ceph_connection *con)
1727 {
1728         int lens[] = {
1729                 sizeof(struct ceph_msg_header2),
1730                 front_len(con->out_msg),
1731                 middle_len(con->out_msg),
1732                 data_len(con->out_msg)
1733         };
1734         struct ceph_frame_desc desc;
1735         int ret;
1736
1737         dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1738              con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1739
1740         if (con->in_seq > con->in_seq_acked) {
1741                 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1742                      con->in_seq_acked, con->in_seq);
1743                 con->in_seq_acked = con->in_seq;
1744         }
1745
1746         reset_out_kvecs(con);
1747         init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1748         encode_preamble(&desc, con->v2.out_buf);
1749         fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1750                      con->in_seq_acked);
1751
1752         if (con_secure(con)) {
1753                 ret = prepare_message_secure(con);
1754                 if (ret)
1755                         return ret;
1756         } else {
1757                 prepare_message_plain(con);
1758         }
1759
1760         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1761         return 0;
1762 }
1763
1764 static int prepare_read_banner_prefix(struct ceph_connection *con)
1765 {
1766         void *buf;
1767
1768         buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1769         if (!buf)
1770                 return -ENOMEM;
1771
1772         reset_in_kvecs(con);
1773         add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1774         add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1775         con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1776         return 0;
1777 }
1778
1779 static int prepare_read_banner_payload(struct ceph_connection *con,
1780                                        int payload_len)
1781 {
1782         void *buf;
1783
1784         buf = alloc_conn_buf(con, payload_len);
1785         if (!buf)
1786                 return -ENOMEM;
1787
1788         reset_in_kvecs(con);
1789         add_in_kvec(con, buf, payload_len);
1790         add_in_sign_kvec(con, buf, payload_len);
1791         con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1792         return 0;
1793 }
1794
1795 static void prepare_read_preamble(struct ceph_connection *con)
1796 {
1797         reset_in_kvecs(con);
1798         add_in_kvec(con, con->v2.in_buf,
1799                     con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1800                                       CEPH_PREAMBLE_PLAIN_LEN);
1801         con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1802 }
1803
1804 static int prepare_read_control(struct ceph_connection *con)
1805 {
1806         int ctrl_len = con->v2.in_desc.fd_lens[0];
1807         int head_len;
1808         void *buf;
1809
1810         reset_in_kvecs(con);
1811         if (con->state == CEPH_CON_S_V2_HELLO ||
1812             con->state == CEPH_CON_S_V2_AUTH) {
1813                 head_len = head_onwire_len(ctrl_len, false);
1814                 buf = alloc_conn_buf(con, head_len);
1815                 if (!buf)
1816                         return -ENOMEM;
1817
1818                 /* preserve preamble */
1819                 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1820
1821                 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1822                 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1823                 add_in_sign_kvec(con, buf, head_len);
1824         } else {
1825                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1826                         buf = alloc_conn_buf(con, ctrl_len);
1827                         if (!buf)
1828                                 return -ENOMEM;
1829
1830                         add_in_kvec(con, buf, ctrl_len);
1831                 } else {
1832                         add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1833                 }
1834                 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1835         }
1836         con->v2.in_state = IN_S_HANDLE_CONTROL;
1837         return 0;
1838 }
1839
1840 static int prepare_read_control_remainder(struct ceph_connection *con)
1841 {
1842         int ctrl_len = con->v2.in_desc.fd_lens[0];
1843         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1844         void *buf;
1845
1846         buf = alloc_conn_buf(con, ctrl_len);
1847         if (!buf)
1848                 return -ENOMEM;
1849
1850         memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1851
1852         reset_in_kvecs(con);
1853         add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1854         add_in_kvec(con, con->v2.in_buf,
1855                     padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1856         con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1857         return 0;
1858 }
1859
1860 static int prepare_read_data(struct ceph_connection *con)
1861 {
1862         struct bio_vec bv;
1863
1864         con->in_data_crc = -1;
1865         ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1866                                   data_len(con->in_msg));
1867
1868         get_bvec_at(&con->v2.in_cursor, &bv);
1869         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1870                 if (unlikely(!con->bounce_page)) {
1871                         con->bounce_page = alloc_page(GFP_NOIO);
1872                         if (!con->bounce_page) {
1873                                 pr_err("failed to allocate bounce page\n");
1874                                 return -ENOMEM;
1875                         }
1876                 }
1877
1878                 bv.bv_page = con->bounce_page;
1879                 bv.bv_offset = 0;
1880         }
1881         set_in_bvec(con, &bv);
1882         con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1883         return 0;
1884 }
1885
1886 static void prepare_read_data_cont(struct ceph_connection *con)
1887 {
1888         struct bio_vec bv;
1889
1890         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1891                 con->in_data_crc = crc32c(con->in_data_crc,
1892                                           page_address(con->bounce_page),
1893                                           con->v2.in_bvec.bv_len);
1894
1895                 get_bvec_at(&con->v2.in_cursor, &bv);
1896                 memcpy_to_page(bv.bv_page, bv.bv_offset,
1897                                page_address(con->bounce_page),
1898                                con->v2.in_bvec.bv_len);
1899         } else {
1900                 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1901                                                     con->v2.in_bvec.bv_page,
1902                                                     con->v2.in_bvec.bv_offset,
1903                                                     con->v2.in_bvec.bv_len);
1904         }
1905
1906         ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1907         if (con->v2.in_cursor.total_resid) {
1908                 get_bvec_at(&con->v2.in_cursor, &bv);
1909                 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1910                         bv.bv_page = con->bounce_page;
1911                         bv.bv_offset = 0;
1912                 }
1913                 set_in_bvec(con, &bv);
1914                 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1915                 return;
1916         }
1917
1918         /*
1919          * We've read all data.  Prepare to read epilogue.
1920          */
1921         reset_in_kvecs(con);
1922         add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1923         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1924 }
1925
1926 static int prepare_sparse_read_cont(struct ceph_connection *con)
1927 {
1928         int ret;
1929         struct bio_vec bv;
1930         char *buf = NULL;
1931         struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1932
1933         WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);
1934
1935         if (iov_iter_is_bvec(&con->v2.in_iter)) {
1936                 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1937                         con->in_data_crc = crc32c(con->in_data_crc,
1938                                                   page_address(con->bounce_page),
1939                                                   con->v2.in_bvec.bv_len);
1940                         get_bvec_at(cursor, &bv);
1941                         memcpy_to_page(bv.bv_page, bv.bv_offset,
1942                                        page_address(con->bounce_page),
1943                                        con->v2.in_bvec.bv_len);
1944                 } else {
1945                         con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1946                                                             con->v2.in_bvec.bv_page,
1947                                                             con->v2.in_bvec.bv_offset,
1948                                                             con->v2.in_bvec.bv_len);
1949                 }
1950
1951                 ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);
1952                 cursor->sr_resid -= con->v2.in_bvec.bv_len;
1953                 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__,
1954                      con->v2.in_bvec.bv_len, cursor->sr_resid);
1955                 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid);
1956                 if (cursor->sr_resid) {
1957                         get_bvec_at(cursor, &bv);
1958                         if (bv.bv_len > cursor->sr_resid)
1959                                 bv.bv_len = cursor->sr_resid;
1960                         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1961                                 bv.bv_page = con->bounce_page;
1962                                 bv.bv_offset = 0;
1963                         }
1964                         set_in_bvec(con, &bv);
1965                         con->v2.data_len_remain -= bv.bv_len;
1966                         return 0;
1967                 }
1968         } else if (iov_iter_is_kvec(&con->v2.in_iter)) {
1969                 /* On first call, we have no kvec so don't compute crc */
1970                 if (con->v2.in_kvec_cnt) {
1971                         WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);
1972                         con->in_data_crc = crc32c(con->in_data_crc,
1973                                                   con->v2.in_kvecs[0].iov_base,
1974                                                   con->v2.in_kvecs[0].iov_len);
1975                 }
1976         } else {
1977                 return -EIO;
1978         }
1979
1980         /* get next extent */
1981         ret = con->ops->sparse_read(con, cursor, &buf);
1982         if (ret <= 0) {
1983                 if (ret < 0)
1984                         return ret;
1985
1986                 reset_in_kvecs(con);
1987                 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1988                 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1989                 return 0;
1990         }
1991
1992         if (buf) {
1993                 /* receive into buffer */
1994                 reset_in_kvecs(con);
1995                 add_in_kvec(con, buf, ret);
1996                 con->v2.data_len_remain -= ret;
1997                 return 0;
1998         }
1999
2000         if (ret > cursor->total_resid) {
2001                 pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n",
2002                         __func__, ret, cursor->total_resid, cursor->resid);
2003                 return -EIO;
2004         }
2005         get_bvec_at(cursor, &bv);
2006         if (bv.bv_len > cursor->sr_resid)
2007                 bv.bv_len = cursor->sr_resid;
2008         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
2009                 if (unlikely(!con->bounce_page)) {
2010                         con->bounce_page = alloc_page(GFP_NOIO);
2011                         if (!con->bounce_page) {
2012                                 pr_err("failed to allocate bounce page\n");
2013                                 return -ENOMEM;
2014                         }
2015                 }
2016
2017                 bv.bv_page = con->bounce_page;
2018                 bv.bv_offset = 0;
2019         }
2020         set_in_bvec(con, &bv);
2021         con->v2.data_len_remain -= ret;
2022         return ret;
2023 }
2024
2025 static int prepare_sparse_read_data(struct ceph_connection *con)
2026 {
2027         struct ceph_msg *msg = con->in_msg;
2028
2029         dout("%s: starting sparse read\n", __func__);
2030
2031         if (WARN_ON_ONCE(!con->ops->sparse_read))
2032                 return -EOPNOTSUPP;
2033
2034         if (!con_secure(con))
2035                 con->in_data_crc = -1;
2036
2037         reset_in_kvecs(con);
2038         con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;
2039         con->v2.data_len_remain = data_len(msg);
2040         return prepare_sparse_read_cont(con);
2041 }
2042
2043 static int prepare_read_tail_plain(struct ceph_connection *con)
2044 {
2045         struct ceph_msg *msg = con->in_msg;
2046
2047         if (!front_len(msg) && !middle_len(msg)) {
2048                 WARN_ON(!data_len(msg));
2049                 return prepare_read_data(con);
2050         }
2051
2052         reset_in_kvecs(con);
2053         if (front_len(msg)) {
2054                 add_in_kvec(con, msg->front.iov_base, front_len(msg));
2055                 WARN_ON(msg->front.iov_len != front_len(msg));
2056         }
2057         if (middle_len(msg)) {
2058                 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
2059                 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
2060         }
2061
2062         if (data_len(msg)) {
2063                 if (msg->sparse_read)
2064                         con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;
2065                 else
2066                         con->v2.in_state = IN_S_PREPARE_READ_DATA;
2067         } else {
2068                 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
2069                 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2070         }
2071         return 0;
2072 }
2073
2074 static void prepare_read_enc_page(struct ceph_connection *con)
2075 {
2076         struct bio_vec bv;
2077
2078         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
2079              con->v2.in_enc_resid);
2080         WARN_ON(!con->v2.in_enc_resid);
2081
2082         bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],
2083                       min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);
2084
2085         set_in_bvec(con, &bv);
2086         con->v2.in_enc_i++;
2087         con->v2.in_enc_resid -= bv.bv_len;
2088
2089         if (con->v2.in_enc_resid) {
2090                 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
2091                 return;
2092         }
2093
2094         /*
2095          * We are set to read the last piece of ciphertext (ending
2096          * with epilogue) + auth tag.
2097          */
2098         WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
2099         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2100 }
2101
2102 static int prepare_read_tail_secure(struct ceph_connection *con)
2103 {
2104         struct page **enc_pages;
2105         int enc_page_cnt;
2106         int tail_len;
2107
2108         tail_len = tail_onwire_len(con->in_msg, true);
2109         WARN_ON(!tail_len);
2110
2111         enc_page_cnt = calc_pages_for(0, tail_len);
2112         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
2113         if (IS_ERR(enc_pages))
2114                 return PTR_ERR(enc_pages);
2115
2116         WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
2117         con->v2.in_enc_pages = enc_pages;
2118         con->v2.in_enc_page_cnt = enc_page_cnt;
2119         con->v2.in_enc_resid = tail_len;
2120         con->v2.in_enc_i = 0;
2121
2122         prepare_read_enc_page(con);
2123         return 0;
2124 }
2125
2126 static void __finish_skip(struct ceph_connection *con)
2127 {
2128         con->in_seq++;
2129         prepare_read_preamble(con);
2130 }
2131
2132 static void prepare_skip_message(struct ceph_connection *con)
2133 {
2134         struct ceph_frame_desc *desc = &con->v2.in_desc;
2135         int tail_len;
2136
2137         dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
2138              desc->fd_lens[2], desc->fd_lens[3]);
2139
2140         tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
2141                                      desc->fd_lens[3], con_secure(con));
2142         if (!tail_len) {
2143                 __finish_skip(con);
2144         } else {
2145                 set_in_skip(con, tail_len);
2146                 con->v2.in_state = IN_S_FINISH_SKIP;
2147         }
2148 }
2149
2150 static int process_banner_prefix(struct ceph_connection *con)
2151 {
2152         int payload_len;
2153         void *p;
2154
2155         WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
2156
2157         p = con->v2.in_kvecs[0].iov_base;
2158         if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
2159                 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
2160                         con->error_msg = "server is speaking msgr1 protocol";
2161                 else
2162                         con->error_msg = "protocol error, bad banner";
2163                 return -EINVAL;
2164         }
2165
2166         p += CEPH_BANNER_V2_LEN;
2167         payload_len = ceph_decode_16(&p);
2168         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2169
2170         return prepare_read_banner_payload(con, payload_len);
2171 }
2172
2173 static int process_banner_payload(struct ceph_connection *con)
2174 {
2175         void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
2176         u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
2177         u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
2178         u64 server_feat, server_req_feat;
2179         void *p;
2180         int ret;
2181
2182         p = con->v2.in_kvecs[0].iov_base;
2183         ceph_decode_64_safe(&p, end, server_feat, bad);
2184         ceph_decode_64_safe(&p, end, server_req_feat, bad);
2185
2186         dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
2187              __func__, con, server_feat, server_req_feat);
2188
2189         if (req_feat & ~server_feat) {
2190                 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2191                        server_feat, req_feat & ~server_feat);
2192                 con->error_msg = "missing required protocol features";
2193                 return -EINVAL;
2194         }
2195         if (server_req_feat & ~feat) {
2196                 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2197                        feat, server_req_feat & ~feat);
2198                 con->error_msg = "missing required protocol features";
2199                 return -EINVAL;
2200         }
2201
2202         /* no reset_out_kvecs() as our banner may still be pending */
2203         ret = prepare_hello(con);
2204         if (ret) {
2205                 pr_err("prepare_hello failed: %d\n", ret);
2206                 return ret;
2207         }
2208
2209         con->state = CEPH_CON_S_V2_HELLO;
2210         prepare_read_preamble(con);
2211         return 0;
2212
2213 bad:
2214         pr_err("failed to decode banner payload\n");
2215         return -EINVAL;
2216 }
2217
2218 static int process_hello(struct ceph_connection *con, void *p, void *end)
2219 {
2220         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2221         struct ceph_entity_addr addr_for_me;
2222         u8 entity_type;
2223         int ret;
2224
2225         if (con->state != CEPH_CON_S_V2_HELLO) {
2226                 con->error_msg = "protocol error, unexpected hello";
2227                 return -EINVAL;
2228         }
2229
2230         ceph_decode_8_safe(&p, end, entity_type, bad);
2231         ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
2232         if (ret) {
2233                 pr_err("failed to decode addr_for_me: %d\n", ret);
2234                 return ret;
2235         }
2236
2237         dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2238              entity_type, ceph_pr_addr(&addr_for_me));
2239
2240         if (entity_type != con->peer_name.type) {
2241                 pr_err("bad peer type, want %d, got %d\n",
2242                        con->peer_name.type, entity_type);
2243                 con->error_msg = "wrong peer at address";
2244                 return -EINVAL;
2245         }
2246
2247         /*
2248          * Set our address to the address our first peer (i.e. monitor)
2249          * sees that we are connecting from.  If we are behind some sort
2250          * of NAT and want to be identified by some private (not NATed)
2251          * address, ip option should be used.
2252          */
2253         if (ceph_addr_is_blank(my_addr)) {
2254                 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2255                        sizeof(my_addr->in_addr));
2256                 ceph_addr_set_port(my_addr, 0);
2257                 dout("%s con %p set my addr %s, as seen by peer %s\n",
2258                      __func__, con, ceph_pr_addr(my_addr),
2259                      ceph_pr_addr(&con->peer_addr));
2260         } else {
2261                 dout("%s con %p my addr already set %s\n",
2262                      __func__, con, ceph_pr_addr(my_addr));
2263         }
2264
2265         WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
2266         WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2267         WARN_ON(!my_addr->nonce);
2268
2269         /* no reset_out_kvecs() as our hello may still be pending */
2270         ret = prepare_auth_request(con);
2271         if (ret) {
2272                 if (ret != -EAGAIN)
2273                         pr_err("prepare_auth_request failed: %d\n", ret);
2274                 return ret;
2275         }
2276
2277         con->state = CEPH_CON_S_V2_AUTH;
2278         return 0;
2279
2280 bad:
2281         pr_err("failed to decode hello\n");
2282         return -EINVAL;
2283 }
2284
2285 static int process_auth_bad_method(struct ceph_connection *con,
2286                                    void *p, void *end)
2287 {
2288         int allowed_protos[8], allowed_modes[8];
2289         int allowed_proto_cnt, allowed_mode_cnt;
2290         int used_proto, result;
2291         int ret;
2292         int i;
2293
2294         if (con->state != CEPH_CON_S_V2_AUTH) {
2295                 con->error_msg = "protocol error, unexpected auth_bad_method";
2296                 return -EINVAL;
2297         }
2298
2299         ceph_decode_32_safe(&p, end, used_proto, bad);
2300         ceph_decode_32_safe(&p, end, result, bad);
2301         dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2302              result);
2303
2304         ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2305         if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2306                 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2307                 return -EINVAL;
2308         }
2309         for (i = 0; i < allowed_proto_cnt; i++) {
2310                 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2311                 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2312                      i, allowed_protos[i]);
2313         }
2314
2315         ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2316         if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2317                 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2318                 return -EINVAL;
2319         }
2320         for (i = 0; i < allowed_mode_cnt; i++) {
2321                 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2322                 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2323                      i, allowed_modes[i]);
2324         }
2325
2326         mutex_unlock(&con->mutex);
2327         ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2328                                                allowed_protos,
2329                                                allowed_proto_cnt,
2330                                                allowed_modes,
2331                                                allowed_mode_cnt);
2332         mutex_lock(&con->mutex);
2333         if (con->state != CEPH_CON_S_V2_AUTH) {
2334                 dout("%s con %p state changed to %d\n", __func__, con,
2335                      con->state);
2336                 return -EAGAIN;
2337         }
2338
2339         dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2340         return ret;
2341
2342 bad:
2343         pr_err("failed to decode auth_bad_method\n");
2344         return -EINVAL;
2345 }
2346
2347 static int process_auth_reply_more(struct ceph_connection *con,
2348                                    void *p, void *end)
2349 {
2350         int payload_len;
2351         int ret;
2352
2353         if (con->state != CEPH_CON_S_V2_AUTH) {
2354                 con->error_msg = "protocol error, unexpected auth_reply_more";
2355                 return -EINVAL;
2356         }
2357
2358         ceph_decode_32_safe(&p, end, payload_len, bad);
2359         ceph_decode_need(&p, end, payload_len, bad);
2360
2361         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2362
2363         reset_out_kvecs(con);
2364         ret = prepare_auth_request_more(con, p, payload_len);
2365         if (ret) {
2366                 if (ret != -EAGAIN)
2367                         pr_err("prepare_auth_request_more failed: %d\n", ret);
2368                 return ret;
2369         }
2370
2371         return 0;
2372
2373 bad:
2374         pr_err("failed to decode auth_reply_more\n");
2375         return -EINVAL;
2376 }
2377
2378 /*
2379  * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2380  * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2381  * setup_crypto().  __aligned(16) isn't guaranteed to work for stack
2382  * objects, so do it by hand.
2383  */
2384 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2385 {
2386         u8 session_key_buf[CEPH_KEY_LEN + 16];
2387         u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2388         u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2389         u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2390         int session_key_len, con_secret_len;
2391         int payload_len;
2392         u64 global_id;
2393         int ret;
2394
2395         if (con->state != CEPH_CON_S_V2_AUTH) {
2396                 con->error_msg = "protocol error, unexpected auth_done";
2397                 return -EINVAL;
2398         }
2399
2400         ceph_decode_64_safe(&p, end, global_id, bad);
2401         ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2402         ceph_decode_32_safe(&p, end, payload_len, bad);
2403
2404         dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2405              __func__, con, global_id, con->v2.con_mode, payload_len);
2406
2407         mutex_unlock(&con->mutex);
2408         session_key_len = 0;
2409         con_secret_len = 0;
2410         ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2411                                          session_key, &session_key_len,
2412                                          con_secret, &con_secret_len);
2413         mutex_lock(&con->mutex);
2414         if (con->state != CEPH_CON_S_V2_AUTH) {
2415                 dout("%s con %p state changed to %d\n", __func__, con,
2416                      con->state);
2417                 ret = -EAGAIN;
2418                 goto out;
2419         }
2420
2421         dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2422         if (ret)
2423                 goto out;
2424
2425         ret = setup_crypto(con, session_key, session_key_len, con_secret,
2426                            con_secret_len);
2427         if (ret)
2428                 goto out;
2429
2430         reset_out_kvecs(con);
2431         ret = prepare_auth_signature(con);
2432         if (ret) {
2433                 pr_err("prepare_auth_signature failed: %d\n", ret);
2434                 goto out;
2435         }
2436
2437         con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2438
2439 out:
2440         memzero_explicit(session_key_buf, sizeof(session_key_buf));
2441         memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2442         return ret;
2443
2444 bad:
2445         pr_err("failed to decode auth_done\n");
2446         return -EINVAL;
2447 }
2448
2449 static int process_auth_signature(struct ceph_connection *con,
2450                                   void *p, void *end)
2451 {
2452         u8 hmac[SHA256_DIGEST_SIZE];
2453         int ret;
2454
2455         if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2456                 con->error_msg = "protocol error, unexpected auth_signature";
2457                 return -EINVAL;
2458         }
2459
2460         ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2461                           con->v2.out_sign_kvec_cnt, hmac);
2462         if (ret)
2463                 return ret;
2464
2465         ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2466         if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2467                 con->error_msg = "integrity error, bad auth signature";
2468                 return -EBADMSG;
2469         }
2470
2471         dout("%s con %p auth signature ok\n", __func__, con);
2472
2473         /* no reset_out_kvecs() as our auth_signature may still be pending */
2474         if (!con->v2.server_cookie) {
2475                 ret = prepare_client_ident(con);
2476                 if (ret) {
2477                         pr_err("prepare_client_ident failed: %d\n", ret);
2478                         return ret;
2479                 }
2480
2481                 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2482         } else {
2483                 ret = prepare_session_reconnect(con);
2484                 if (ret) {
2485                         pr_err("prepare_session_reconnect failed: %d\n", ret);
2486                         return ret;
2487                 }
2488
2489                 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2490         }
2491
2492         return 0;
2493
2494 bad:
2495         pr_err("failed to decode auth_signature\n");
2496         return -EINVAL;
2497 }
2498
2499 static int process_server_ident(struct ceph_connection *con,
2500                                 void *p, void *end)
2501 {
2502         struct ceph_client *client = from_msgr(con->msgr);
2503         u64 features, required_features;
2504         struct ceph_entity_addr addr;
2505         u64 global_seq;
2506         u64 global_id;
2507         u64 cookie;
2508         u64 flags;
2509         int ret;
2510
2511         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2512                 con->error_msg = "protocol error, unexpected server_ident";
2513                 return -EINVAL;
2514         }
2515
2516         ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2517         if (ret) {
2518                 pr_err("failed to decode server addrs: %d\n", ret);
2519                 return ret;
2520         }
2521
2522         ceph_decode_64_safe(&p, end, global_id, bad);
2523         ceph_decode_64_safe(&p, end, global_seq, bad);
2524         ceph_decode_64_safe(&p, end, features, bad);
2525         ceph_decode_64_safe(&p, end, required_features, bad);
2526         ceph_decode_64_safe(&p, end, flags, bad);
2527         ceph_decode_64_safe(&p, end, cookie, bad);
2528
2529         dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2530              __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2531              global_id, global_seq, features, required_features, flags, cookie);
2532
2533         /* is this who we intended to talk to? */
2534         if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2535                 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2536                        ceph_pr_addr(&con->peer_addr),
2537                        le32_to_cpu(con->peer_addr.nonce),
2538                        ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2539                 con->error_msg = "wrong peer at address";
2540                 return -EINVAL;
2541         }
2542
2543         if (client->required_features & ~features) {
2544                 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2545                        features, client->required_features & ~features);
2546                 con->error_msg = "missing required protocol features";
2547                 return -EINVAL;
2548         }
2549
2550         /*
2551          * Both name->type and name->num are set in ceph_con_open() but
2552          * name->num may be bogus in the initial monmap.  name->type is
2553          * verified in handle_hello().
2554          */
2555         WARN_ON(!con->peer_name.type);
2556         con->peer_name.num = cpu_to_le64(global_id);
2557         con->v2.peer_global_seq = global_seq;
2558         con->peer_features = features;
2559         WARN_ON(required_features & ~client->supported_features);
2560         con->v2.server_cookie = cookie;
2561
2562         if (flags & CEPH_MSG_CONNECT_LOSSY) {
2563                 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2564                 WARN_ON(con->v2.server_cookie);
2565         } else {
2566                 WARN_ON(!con->v2.server_cookie);
2567         }
2568
2569         clear_in_sign_kvecs(con);
2570         clear_out_sign_kvecs(con);
2571         free_conn_bufs(con);
2572         con->delay = 0;  /* reset backoff memory */
2573
2574         con->state = CEPH_CON_S_OPEN;
2575         con->v2.out_state = OUT_S_GET_NEXT;
2576         return 0;
2577
2578 bad:
2579         pr_err("failed to decode server_ident\n");
2580         return -EINVAL;
2581 }
2582
2583 static int process_ident_missing_features(struct ceph_connection *con,
2584                                           void *p, void *end)
2585 {
2586         struct ceph_client *client = from_msgr(con->msgr);
2587         u64 missing_features;
2588
2589         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2590                 con->error_msg = "protocol error, unexpected ident_missing_features";
2591                 return -EINVAL;
2592         }
2593
2594         ceph_decode_64_safe(&p, end, missing_features, bad);
2595         pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2596                client->supported_features, missing_features);
2597         con->error_msg = "missing required protocol features";
2598         return -EINVAL;
2599
2600 bad:
2601         pr_err("failed to decode ident_missing_features\n");
2602         return -EINVAL;
2603 }
2604
2605 static int process_session_reconnect_ok(struct ceph_connection *con,
2606                                         void *p, void *end)
2607 {
2608         u64 seq;
2609
2610         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2611                 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2612                 return -EINVAL;
2613         }
2614
2615         ceph_decode_64_safe(&p, end, seq, bad);
2616
2617         dout("%s con %p seq %llu\n", __func__, con, seq);
2618         ceph_con_discard_requeued(con, seq);
2619
2620         clear_in_sign_kvecs(con);
2621         clear_out_sign_kvecs(con);
2622         free_conn_bufs(con);
2623         con->delay = 0;  /* reset backoff memory */
2624
2625         con->state = CEPH_CON_S_OPEN;
2626         con->v2.out_state = OUT_S_GET_NEXT;
2627         return 0;
2628
2629 bad:
2630         pr_err("failed to decode session_reconnect_ok\n");
2631         return -EINVAL;
2632 }
2633
2634 static int process_session_retry(struct ceph_connection *con,
2635                                  void *p, void *end)
2636 {
2637         u64 connect_seq;
2638         int ret;
2639
2640         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2641                 con->error_msg = "protocol error, unexpected session_retry";
2642                 return -EINVAL;
2643         }
2644
2645         ceph_decode_64_safe(&p, end, connect_seq, bad);
2646
2647         dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2648         WARN_ON(connect_seq <= con->v2.connect_seq);
2649         con->v2.connect_seq = connect_seq + 1;
2650
2651         free_conn_bufs(con);
2652
2653         reset_out_kvecs(con);
2654         ret = prepare_session_reconnect(con);
2655         if (ret) {
2656                 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2657                 return ret;
2658         }
2659
2660         return 0;
2661
2662 bad:
2663         pr_err("failed to decode session_retry\n");
2664         return -EINVAL;
2665 }
2666
2667 static int process_session_retry_global(struct ceph_connection *con,
2668                                         void *p, void *end)
2669 {
2670         u64 global_seq;
2671         int ret;
2672
2673         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2674                 con->error_msg = "protocol error, unexpected session_retry_global";
2675                 return -EINVAL;
2676         }
2677
2678         ceph_decode_64_safe(&p, end, global_seq, bad);
2679
2680         dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2681         WARN_ON(global_seq <= con->v2.global_seq);
2682         con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2683
2684         free_conn_bufs(con);
2685
2686         reset_out_kvecs(con);
2687         ret = prepare_session_reconnect(con);
2688         if (ret) {
2689                 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2690                 return ret;
2691         }
2692
2693         return 0;
2694
2695 bad:
2696         pr_err("failed to decode session_retry_global\n");
2697         return -EINVAL;
2698 }
2699
2700 static int process_session_reset(struct ceph_connection *con,
2701                                  void *p, void *end)
2702 {
2703         bool full;
2704         int ret;
2705
2706         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2707                 con->error_msg = "protocol error, unexpected session_reset";
2708                 return -EINVAL;
2709         }
2710
2711         ceph_decode_8_safe(&p, end, full, bad);
2712         if (!full) {
2713                 con->error_msg = "protocol error, bad session_reset";
2714                 return -EINVAL;
2715         }
2716
2717         pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2718                 ceph_pr_addr(&con->peer_addr));
2719         ceph_con_reset_session(con);
2720
2721         mutex_unlock(&con->mutex);
2722         if (con->ops->peer_reset)
2723                 con->ops->peer_reset(con);
2724         mutex_lock(&con->mutex);
2725         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2726                 dout("%s con %p state changed to %d\n", __func__, con,
2727                      con->state);
2728                 return -EAGAIN;
2729         }
2730
2731         free_conn_bufs(con);
2732
2733         reset_out_kvecs(con);
2734         ret = prepare_client_ident(con);
2735         if (ret) {
2736                 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2737                 return ret;
2738         }
2739
2740         con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2741         return 0;
2742
2743 bad:
2744         pr_err("failed to decode session_reset\n");
2745         return -EINVAL;
2746 }
2747
2748 static int process_keepalive2_ack(struct ceph_connection *con,
2749                                   void *p, void *end)
2750 {
2751         if (con->state != CEPH_CON_S_OPEN) {
2752                 con->error_msg = "protocol error, unexpected keepalive2_ack";
2753                 return -EINVAL;
2754         }
2755
2756         ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2757         ceph_decode_timespec64(&con->last_keepalive_ack, p);
2758
2759         dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2760              con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2761
2762         return 0;
2763
2764 bad:
2765         pr_err("failed to decode keepalive2_ack\n");
2766         return -EINVAL;
2767 }
2768
2769 static int process_ack(struct ceph_connection *con, void *p, void *end)
2770 {
2771         u64 seq;
2772
2773         if (con->state != CEPH_CON_S_OPEN) {
2774                 con->error_msg = "protocol error, unexpected ack";
2775                 return -EINVAL;
2776         }
2777
2778         ceph_decode_64_safe(&p, end, seq, bad);
2779
2780         dout("%s con %p seq %llu\n", __func__, con, seq);
2781         ceph_con_discard_sent(con, seq);
2782         return 0;
2783
2784 bad:
2785         pr_err("failed to decode ack\n");
2786         return -EINVAL;
2787 }
2788
2789 static int process_control(struct ceph_connection *con, void *p, void *end)
2790 {
2791         int tag = con->v2.in_desc.fd_tag;
2792         int ret;
2793
2794         dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2795
2796         switch (tag) {
2797         case FRAME_TAG_HELLO:
2798                 ret = process_hello(con, p, end);
2799                 break;
2800         case FRAME_TAG_AUTH_BAD_METHOD:
2801                 ret = process_auth_bad_method(con, p, end);
2802                 break;
2803         case FRAME_TAG_AUTH_REPLY_MORE:
2804                 ret = process_auth_reply_more(con, p, end);
2805                 break;
2806         case FRAME_TAG_AUTH_DONE:
2807                 ret = process_auth_done(con, p, end);
2808                 break;
2809         case FRAME_TAG_AUTH_SIGNATURE:
2810                 ret = process_auth_signature(con, p, end);
2811                 break;
2812         case FRAME_TAG_SERVER_IDENT:
2813                 ret = process_server_ident(con, p, end);
2814                 break;
2815         case FRAME_TAG_IDENT_MISSING_FEATURES:
2816                 ret = process_ident_missing_features(con, p, end);
2817                 break;
2818         case FRAME_TAG_SESSION_RECONNECT_OK:
2819                 ret = process_session_reconnect_ok(con, p, end);
2820                 break;
2821         case FRAME_TAG_SESSION_RETRY:
2822                 ret = process_session_retry(con, p, end);
2823                 break;
2824         case FRAME_TAG_SESSION_RETRY_GLOBAL:
2825                 ret = process_session_retry_global(con, p, end);
2826                 break;
2827         case FRAME_TAG_SESSION_RESET:
2828                 ret = process_session_reset(con, p, end);
2829                 break;
2830         case FRAME_TAG_KEEPALIVE2_ACK:
2831                 ret = process_keepalive2_ack(con, p, end);
2832                 break;
2833         case FRAME_TAG_ACK:
2834                 ret = process_ack(con, p, end);
2835                 break;
2836         default:
2837                 pr_err("bad tag %d\n", tag);
2838                 con->error_msg = "protocol error, bad tag";
2839                 return -EINVAL;
2840         }
2841         if (ret) {
2842                 dout("%s con %p error %d\n", __func__, con, ret);
2843                 return ret;
2844         }
2845
2846         prepare_read_preamble(con);
2847         return 0;
2848 }
2849
2850 /*
2851  * Return:
2852  *   1 - con->in_msg set, read message
2853  *   0 - skip message
2854  *  <0 - error
2855  */
2856 static int process_message_header(struct ceph_connection *con,
2857                                   void *p, void *end)
2858 {
2859         struct ceph_frame_desc *desc = &con->v2.in_desc;
2860         struct ceph_msg_header2 *hdr2 = p;
2861         struct ceph_msg_header hdr;
2862         int skip;
2863         int ret;
2864         u64 seq;
2865
2866         /* verify seq# */
2867         seq = le64_to_cpu(hdr2->seq);
2868         if ((s64)seq - (s64)con->in_seq < 1) {
2869                 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2870                         ENTITY_NAME(con->peer_name),
2871                         ceph_pr_addr(&con->peer_addr),
2872                         seq, con->in_seq + 1);
2873                 return 0;
2874         }
2875         if ((s64)seq - (s64)con->in_seq > 1) {
2876                 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2877                 con->error_msg = "bad message sequence # for incoming message";
2878                 return -EBADE;
2879         }
2880
2881         ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2882
2883         fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2884                     desc->fd_lens[3], &con->peer_name);
2885         ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2886         if (ret)
2887                 return ret;
2888
2889         WARN_ON(!con->in_msg ^ skip);
2890         if (skip)
2891                 return 0;
2892
2893         WARN_ON(!con->in_msg);
2894         WARN_ON(con->in_msg->con != con);
2895         return 1;
2896 }
2897
2898 static int process_message(struct ceph_connection *con)
2899 {
2900         ceph_con_process_message(con);
2901
2902         /*
2903          * We could have been closed by ceph_con_close() because
2904          * ceph_con_process_message() temporarily drops con->mutex.
2905          */
2906         if (con->state != CEPH_CON_S_OPEN) {
2907                 dout("%s con %p state changed to %d\n", __func__, con,
2908                      con->state);
2909                 return -EAGAIN;
2910         }
2911
2912         prepare_read_preamble(con);
2913         return 0;
2914 }
2915
2916 static int __handle_control(struct ceph_connection *con, void *p)
2917 {
2918         void *end = p + con->v2.in_desc.fd_lens[0];
2919         struct ceph_msg *msg;
2920         int ret;
2921
2922         if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2923                 return process_control(con, p, end);
2924
2925         ret = process_message_header(con, p, end);
2926         if (ret < 0)
2927                 return ret;
2928         if (ret == 0) {
2929                 prepare_skip_message(con);
2930                 return 0;
2931         }
2932
2933         msg = con->in_msg;  /* set in process_message_header() */
2934         if (front_len(msg)) {
2935                 WARN_ON(front_len(msg) > msg->front_alloc_len);
2936                 msg->front.iov_len = front_len(msg);
2937         } else {
2938                 msg->front.iov_len = 0;
2939         }
2940         if (middle_len(msg)) {
2941                 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2942                 msg->middle->vec.iov_len = middle_len(msg);
2943         } else if (msg->middle) {
2944                 msg->middle->vec.iov_len = 0;
2945         }
2946
2947         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2948                 return process_message(con);
2949
2950         if (con_secure(con))
2951                 return prepare_read_tail_secure(con);
2952
2953         return prepare_read_tail_plain(con);
2954 }
2955
2956 static int handle_preamble(struct ceph_connection *con)
2957 {
2958         struct ceph_frame_desc *desc = &con->v2.in_desc;
2959         int ret;
2960
2961         if (con_secure(con)) {
2962                 ret = decrypt_preamble(con);
2963                 if (ret) {
2964                         if (ret == -EBADMSG)
2965                                 con->error_msg = "integrity error, bad preamble auth tag";
2966                         return ret;
2967                 }
2968         }
2969
2970         ret = decode_preamble(con->v2.in_buf, desc);
2971         if (ret) {
2972                 if (ret == -EBADMSG)
2973                         con->error_msg = "integrity error, bad crc";
2974                 else
2975                         con->error_msg = "protocol error, bad preamble";
2976                 return ret;
2977         }
2978
2979         dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2980              con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2981              desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2982
2983         if (!con_secure(con))
2984                 return prepare_read_control(con);
2985
2986         if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2987                 return prepare_read_control_remainder(con);
2988
2989         return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2990 }
2991
2992 static int handle_control(struct ceph_connection *con)
2993 {
2994         int ctrl_len = con->v2.in_desc.fd_lens[0];
2995         void *buf;
2996         int ret;
2997
2998         WARN_ON(con_secure(con));
2999
3000         ret = verify_control_crc(con);
3001         if (ret) {
3002                 con->error_msg = "integrity error, bad crc";
3003                 return ret;
3004         }
3005
3006         if (con->state == CEPH_CON_S_V2_AUTH) {
3007                 buf = alloc_conn_buf(con, ctrl_len);
3008                 if (!buf)
3009                         return -ENOMEM;
3010
3011                 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
3012                 return __handle_control(con, buf);
3013         }
3014
3015         return __handle_control(con, con->v2.in_kvecs[0].iov_base);
3016 }
3017
3018 static int handle_control_remainder(struct ceph_connection *con)
3019 {
3020         int ret;
3021
3022         WARN_ON(!con_secure(con));
3023
3024         ret = decrypt_control_remainder(con);
3025         if (ret) {
3026                 if (ret == -EBADMSG)
3027                         con->error_msg = "integrity error, bad control remainder auth tag";
3028                 return ret;
3029         }
3030
3031         return __handle_control(con, con->v2.in_kvecs[0].iov_base -
3032                                      CEPH_PREAMBLE_INLINE_LEN);
3033 }
3034
3035 static int handle_epilogue(struct ceph_connection *con)
3036 {
3037         u32 front_crc, middle_crc, data_crc;
3038         int ret;
3039
3040         if (con_secure(con)) {
3041                 ret = decrypt_tail(con);
3042                 if (ret) {
3043                         if (ret == -EBADMSG)
3044                                 con->error_msg = "integrity error, bad epilogue auth tag";
3045                         return ret;
3046                 }
3047
3048                 /* just late_status */
3049                 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
3050                 if (ret) {
3051                         con->error_msg = "protocol error, bad epilogue";
3052                         return ret;
3053                 }
3054         } else {
3055                 ret = decode_epilogue(con->v2.in_buf, &front_crc,
3056                                       &middle_crc, &data_crc);
3057                 if (ret) {
3058                         con->error_msg = "protocol error, bad epilogue";
3059                         return ret;
3060                 }
3061
3062                 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
3063                                            data_crc);
3064                 if (ret) {
3065                         con->error_msg = "integrity error, bad crc";
3066                         return ret;
3067                 }
3068         }
3069
3070         return process_message(con);
3071 }
3072
3073 static void finish_skip(struct ceph_connection *con)
3074 {
3075         dout("%s con %p\n", __func__, con);
3076
3077         if (con_secure(con))
3078                 gcm_inc_nonce(&con->v2.in_gcm_nonce);
3079
3080         __finish_skip(con);
3081 }
3082
3083 static int populate_in_iter(struct ceph_connection *con)
3084 {
3085         int ret;
3086
3087         dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
3088              con->v2.in_state);
3089         WARN_ON(iov_iter_count(&con->v2.in_iter));
3090
3091         if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
3092                 ret = process_banner_prefix(con);
3093         } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
3094                 ret = process_banner_payload(con);
3095         } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
3096                     con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
3097                    con->state == CEPH_CON_S_OPEN) {
3098                 switch (con->v2.in_state) {
3099                 case IN_S_HANDLE_PREAMBLE:
3100                         ret = handle_preamble(con);
3101                         break;
3102                 case IN_S_HANDLE_CONTROL:
3103                         ret = handle_control(con);
3104                         break;
3105                 case IN_S_HANDLE_CONTROL_REMAINDER:
3106                         ret = handle_control_remainder(con);
3107                         break;
3108                 case IN_S_PREPARE_READ_DATA:
3109                         ret = prepare_read_data(con);
3110                         break;
3111                 case IN_S_PREPARE_READ_DATA_CONT:
3112                         prepare_read_data_cont(con);
3113                         ret = 0;
3114                         break;
3115                 case IN_S_PREPARE_READ_ENC_PAGE:
3116                         prepare_read_enc_page(con);
3117                         ret = 0;
3118                         break;
3119                 case IN_S_PREPARE_SPARSE_DATA:
3120                         ret = prepare_sparse_read_data(con);
3121                         break;
3122                 case IN_S_PREPARE_SPARSE_DATA_CONT:
3123                         ret = prepare_sparse_read_cont(con);
3124                         break;
3125                 case IN_S_HANDLE_EPILOGUE:
3126                         ret = handle_epilogue(con);
3127                         break;
3128                 case IN_S_FINISH_SKIP:
3129                         finish_skip(con);
3130                         ret = 0;
3131                         break;
3132                 default:
3133                         WARN(1, "bad in_state %d", con->v2.in_state);
3134                         return -EINVAL;
3135                 }
3136         } else {
3137                 WARN(1, "bad state %d", con->state);
3138                 return -EINVAL;
3139         }
3140         if (ret) {
3141                 dout("%s con %p error %d\n", __func__, con, ret);
3142                 return ret;
3143         }
3144
3145         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3146                 return -ENODATA;
3147         dout("%s con %p populated %zu\n", __func__, con,
3148              iov_iter_count(&con->v2.in_iter));
3149         return 1;
3150 }
3151
3152 int ceph_con_v2_try_read(struct ceph_connection *con)
3153 {
3154         int ret;
3155
3156         dout("%s con %p state %d need %zu\n", __func__, con, con->state,
3157              iov_iter_count(&con->v2.in_iter));
3158
3159         if (con->state == CEPH_CON_S_PREOPEN)
3160                 return 0;
3161
3162         /*
3163          * We should always have something pending here.  If not,
3164          * avoid calling populate_in_iter() as if we read something
3165          * (ceph_tcp_recv() would immediately return 1).
3166          */
3167         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3168                 return -ENODATA;
3169
3170         for (;;) {
3171                 ret = ceph_tcp_recv(con);
3172                 if (ret <= 0)
3173                         return ret;
3174
3175                 ret = populate_in_iter(con);
3176                 if (ret <= 0) {
3177                         if (ret && ret != -EAGAIN && !con->error_msg)
3178                                 con->error_msg = "read processing error";
3179                         return ret;
3180                 }
3181         }
3182 }
3183
3184 static void queue_data(struct ceph_connection *con)
3185 {
3186         struct bio_vec bv;
3187
3188         con->v2.out_epil.data_crc = -1;
3189         ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
3190                                   data_len(con->out_msg));
3191
3192         get_bvec_at(&con->v2.out_cursor, &bv);
3193         set_out_bvec(con, &bv, true);
3194         con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
3195 }
3196
3197 static void queue_data_cont(struct ceph_connection *con)
3198 {
3199         struct bio_vec bv;
3200
3201         con->v2.out_epil.data_crc = ceph_crc32c_page(
3202                 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3203                 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
3204
3205         ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
3206         if (con->v2.out_cursor.total_resid) {
3207                 get_bvec_at(&con->v2.out_cursor, &bv);
3208                 set_out_bvec(con, &bv, true);
3209                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
3210                 return;
3211         }
3212
3213         /*
3214          * We've written all data.  Queue epilogue.  Once it's written,
3215          * we are done.
3216          */
3217         reset_out_kvecs(con);
3218         prepare_epilogue_plain(con, false);
3219         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3220 }
3221
3222 static void queue_enc_page(struct ceph_connection *con)
3223 {
3224         struct bio_vec bv;
3225
3226         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3227              con->v2.out_enc_resid);
3228         WARN_ON(!con->v2.out_enc_resid);
3229
3230         bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],
3231                       min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);
3232
3233         set_out_bvec(con, &bv, false);
3234         con->v2.out_enc_i++;
3235         con->v2.out_enc_resid -= bv.bv_len;
3236
3237         if (con->v2.out_enc_resid) {
3238                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3239                 return;
3240         }
3241
3242         /*
3243          * We've queued the last piece of ciphertext (ending with
3244          * epilogue) + auth tag.  Once it's written, we are done.
3245          */
3246         WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3247         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3248 }
3249
3250 static void queue_zeros(struct ceph_connection *con)
3251 {
3252         dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3253
3254         if (con->v2.out_zero) {
3255                 set_out_bvec_zero(con);
3256                 con->v2.out_zero -= con->v2.out_bvec.bv_len;
3257                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3258                 return;
3259         }
3260
3261         /*
3262          * We've zero-filled everything up to epilogue.  Queue epilogue
3263          * with late_status set to ABORTED and crcs adjusted for zeros.
3264          * Once it's written, we are done patching up for the revoke.
3265          */
3266         reset_out_kvecs(con);
3267         prepare_epilogue_plain(con, true);
3268         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3269 }
3270
3271 static void finish_message(struct ceph_connection *con)
3272 {
3273         dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3274
3275         /* we end up here both plain and secure modes */
3276         if (con->v2.out_enc_pages) {
3277                 WARN_ON(!con->v2.out_enc_page_cnt);
3278                 ceph_release_page_vector(con->v2.out_enc_pages,
3279                                          con->v2.out_enc_page_cnt);
3280                 con->v2.out_enc_pages = NULL;
3281                 con->v2.out_enc_page_cnt = 0;
3282         }
3283         /* message may have been revoked */
3284         if (con->out_msg) {
3285                 ceph_msg_put(con->out_msg);
3286                 con->out_msg = NULL;
3287         }
3288
3289         con->v2.out_state = OUT_S_GET_NEXT;
3290 }
3291
3292 static int populate_out_iter(struct ceph_connection *con)
3293 {
3294         int ret;
3295
3296         dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3297              con->v2.out_state);
3298         WARN_ON(iov_iter_count(&con->v2.out_iter));
3299
3300         if (con->state != CEPH_CON_S_OPEN) {
3301                 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3302                         con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3303                 goto nothing_pending;
3304         }
3305
3306         switch (con->v2.out_state) {
3307         case OUT_S_QUEUE_DATA:
3308                 WARN_ON(!con->out_msg);
3309                 queue_data(con);
3310                 goto populated;
3311         case OUT_S_QUEUE_DATA_CONT:
3312                 WARN_ON(!con->out_msg);
3313                 queue_data_cont(con);
3314                 goto populated;
3315         case OUT_S_QUEUE_ENC_PAGE:
3316                 queue_enc_page(con);
3317                 goto populated;
3318         case OUT_S_QUEUE_ZEROS:
3319                 WARN_ON(con->out_msg);  /* revoked */
3320                 queue_zeros(con);
3321                 goto populated;
3322         case OUT_S_FINISH_MESSAGE:
3323                 finish_message(con);
3324                 break;
3325         case OUT_S_GET_NEXT:
3326                 break;
3327         default:
3328                 WARN(1, "bad out_state %d", con->v2.out_state);
3329                 return -EINVAL;
3330         }
3331
3332         WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3333         if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3334                 ret = prepare_keepalive2(con);
3335                 if (ret) {
3336                         pr_err("prepare_keepalive2 failed: %d\n", ret);
3337                         return ret;
3338                 }
3339         } else if (!list_empty(&con->out_queue)) {
3340                 ceph_con_get_out_msg(con);
3341                 ret = prepare_message(con);
3342                 if (ret) {
3343                         pr_err("prepare_message failed: %d\n", ret);
3344                         return ret;
3345                 }
3346         } else if (con->in_seq > con->in_seq_acked) {
3347                 ret = prepare_ack(con);
3348                 if (ret) {
3349                         pr_err("prepare_ack failed: %d\n", ret);
3350                         return ret;
3351                 }
3352         } else {
3353                 goto nothing_pending;
3354         }
3355
3356 populated:
3357         if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3358                 return -ENODATA;
3359         dout("%s con %p populated %zu\n", __func__, con,
3360              iov_iter_count(&con->v2.out_iter));
3361         return 1;
3362
3363 nothing_pending:
3364         WARN_ON(iov_iter_count(&con->v2.out_iter));
3365         dout("%s con %p nothing pending\n", __func__, con);
3366         ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3367         return 0;
3368 }
3369
3370 int ceph_con_v2_try_write(struct ceph_connection *con)
3371 {
3372         int ret;
3373
3374         dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3375              iov_iter_count(&con->v2.out_iter));
3376
3377         /* open the socket first? */
3378         if (con->state == CEPH_CON_S_PREOPEN) {
3379                 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3380
3381                 /*
3382                  * Always bump global_seq.  Bump connect_seq only if
3383                  * there is a session (i.e. we are reconnecting and will
3384                  * send session_reconnect instead of client_ident).
3385                  */
3386                 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3387                 if (con->v2.server_cookie)
3388                         con->v2.connect_seq++;
3389
3390                 ret = prepare_read_banner_prefix(con);
3391                 if (ret) {
3392                         pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3393                         con->error_msg = "connect error";
3394                         return ret;
3395                 }
3396
3397                 reset_out_kvecs(con);
3398                 ret = prepare_banner(con);
3399                 if (ret) {
3400                         pr_err("prepare_banner failed: %d\n", ret);
3401                         con->error_msg = "connect error";
3402                         return ret;
3403                 }
3404
3405                 ret = ceph_tcp_connect(con);
3406                 if (ret) {
3407                         pr_err("ceph_tcp_connect failed: %d\n", ret);
3408                         con->error_msg = "connect error";
3409                         return ret;
3410                 }
3411         }
3412
3413         if (!iov_iter_count(&con->v2.out_iter)) {
3414                 ret = populate_out_iter(con);
3415                 if (ret <= 0) {
3416                         if (ret && ret != -EAGAIN && !con->error_msg)
3417                                 con->error_msg = "write processing error";
3418                         return ret;
3419                 }
3420         }
3421
3422         tcp_sock_set_cork(con->sock->sk, true);
3423         for (;;) {
3424                 ret = ceph_tcp_send(con);
3425                 if (ret <= 0)
3426                         break;
3427
3428                 ret = populate_out_iter(con);
3429                 if (ret <= 0) {
3430                         if (ret && ret != -EAGAIN && !con->error_msg)
3431                                 con->error_msg = "write processing error";
3432                         break;
3433                 }
3434         }
3435
3436         tcp_sock_set_cork(con->sock->sk, false);
3437         return ret;
3438 }
3439
3440 static u32 crc32c_zeros(u32 crc, int zero_len)
3441 {
3442         int len;
3443
3444         while (zero_len) {
3445                 len = min(zero_len, (int)PAGE_SIZE);
3446                 crc = crc32c(crc, page_address(ceph_zero_page), len);
3447                 zero_len -= len;
3448         }
3449
3450         return crc;
3451 }
3452
3453 static void prepare_zero_front(struct ceph_connection *con, int resid)
3454 {
3455         int sent;
3456
3457         WARN_ON(!resid || resid > front_len(con->out_msg));
3458         sent = front_len(con->out_msg) - resid;
3459         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3460
3461         if (sent) {
3462                 con->v2.out_epil.front_crc =
3463                         crc32c(-1, con->out_msg->front.iov_base, sent);
3464                 con->v2.out_epil.front_crc =
3465                         crc32c_zeros(con->v2.out_epil.front_crc, resid);
3466         } else {
3467                 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3468         }
3469
3470         con->v2.out_iter.count -= resid;
3471         out_zero_add(con, resid);
3472 }
3473
3474 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3475 {
3476         int sent;
3477
3478         WARN_ON(!resid || resid > middle_len(con->out_msg));
3479         sent = middle_len(con->out_msg) - resid;
3480         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3481
3482         if (sent) {
3483                 con->v2.out_epil.middle_crc =
3484                         crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3485                 con->v2.out_epil.middle_crc =
3486                         crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3487         } else {
3488                 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3489         }
3490
3491         con->v2.out_iter.count -= resid;
3492         out_zero_add(con, resid);
3493 }
3494
3495 static void prepare_zero_data(struct ceph_connection *con)
3496 {
3497         dout("%s con %p\n", __func__, con);
3498         con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3499         out_zero_add(con, data_len(con->out_msg));
3500 }
3501
3502 static void revoke_at_queue_data(struct ceph_connection *con)
3503 {
3504         int boundary;
3505         int resid;
3506
3507         WARN_ON(!data_len(con->out_msg));
3508         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3509         resid = iov_iter_count(&con->v2.out_iter);
3510
3511         boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3512         if (resid > boundary) {
3513                 resid -= boundary;
3514                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3515                 dout("%s con %p was sending head\n", __func__, con);
3516                 if (front_len(con->out_msg))
3517                         prepare_zero_front(con, front_len(con->out_msg));
3518                 if (middle_len(con->out_msg))
3519                         prepare_zero_middle(con, middle_len(con->out_msg));
3520                 prepare_zero_data(con);
3521                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3522                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3523                 return;
3524         }
3525
3526         boundary = middle_len(con->out_msg);
3527         if (resid > boundary) {
3528                 resid -= boundary;
3529                 dout("%s con %p was sending front\n", __func__, con);
3530                 prepare_zero_front(con, resid);
3531                 if (middle_len(con->out_msg))
3532                         prepare_zero_middle(con, middle_len(con->out_msg));
3533                 prepare_zero_data(con);
3534                 queue_zeros(con);
3535                 return;
3536         }
3537
3538         WARN_ON(!resid);
3539         dout("%s con %p was sending middle\n", __func__, con);
3540         prepare_zero_middle(con, resid);
3541         prepare_zero_data(con);
3542         queue_zeros(con);
3543 }
3544
3545 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3546 {
3547         int sent, resid;  /* current piece of data */
3548
3549         WARN_ON(!data_len(con->out_msg));
3550         WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3551         resid = iov_iter_count(&con->v2.out_iter);
3552         WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3553         sent = con->v2.out_bvec.bv_len - resid;
3554         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3555
3556         if (sent) {
3557                 con->v2.out_epil.data_crc = ceph_crc32c_page(
3558                         con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3559                         con->v2.out_bvec.bv_offset, sent);
3560                 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3561         }
3562         WARN_ON(resid > con->v2.out_cursor.total_resid);
3563         con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3564                                                 con->v2.out_cursor.total_resid);
3565
3566         con->v2.out_iter.count -= resid;
3567         out_zero_add(con, con->v2.out_cursor.total_resid);
3568         queue_zeros(con);
3569 }
3570
3571 static void revoke_at_finish_message(struct ceph_connection *con)
3572 {
3573         int boundary;
3574         int resid;
3575
3576         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3577         resid = iov_iter_count(&con->v2.out_iter);
3578
3579         if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3580             !data_len(con->out_msg)) {
3581                 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3582                 dout("%s con %p was sending head (empty message) - noop\n",
3583                      __func__, con);
3584                 return;
3585         }
3586
3587         boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3588                    CEPH_EPILOGUE_PLAIN_LEN;
3589         if (resid > boundary) {
3590                 resid -= boundary;
3591                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3592                 dout("%s con %p was sending head\n", __func__, con);
3593                 if (front_len(con->out_msg))
3594                         prepare_zero_front(con, front_len(con->out_msg));
3595                 if (middle_len(con->out_msg))
3596                         prepare_zero_middle(con, middle_len(con->out_msg));
3597                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3598                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3599                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3600                 return;
3601         }
3602
3603         boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3604         if (resid > boundary) {
3605                 resid -= boundary;
3606                 dout("%s con %p was sending front\n", __func__, con);
3607                 prepare_zero_front(con, resid);
3608                 if (middle_len(con->out_msg))
3609                         prepare_zero_middle(con, middle_len(con->out_msg));
3610                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3611                 queue_zeros(con);
3612                 return;
3613         }
3614
3615         boundary = CEPH_EPILOGUE_PLAIN_LEN;
3616         if (resid > boundary) {
3617                 resid -= boundary;
3618                 dout("%s con %p was sending middle\n", __func__, con);
3619                 prepare_zero_middle(con, resid);
3620                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3621                 queue_zeros(con);
3622                 return;
3623         }
3624
3625         WARN_ON(!resid);
3626         dout("%s con %p was sending epilogue - noop\n", __func__, con);
3627 }
3628
3629 void ceph_con_v2_revoke(struct ceph_connection *con)
3630 {
3631         WARN_ON(con->v2.out_zero);
3632
3633         if (con_secure(con)) {
3634                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3635                         con->v2.out_state != OUT_S_FINISH_MESSAGE);
3636                 dout("%s con %p secure - noop\n", __func__, con);
3637                 return;
3638         }
3639
3640         switch (con->v2.out_state) {
3641         case OUT_S_QUEUE_DATA:
3642                 revoke_at_queue_data(con);
3643                 break;
3644         case OUT_S_QUEUE_DATA_CONT:
3645                 revoke_at_queue_data_cont(con);
3646                 break;
3647         case OUT_S_FINISH_MESSAGE:
3648                 revoke_at_finish_message(con);
3649                 break;
3650         default:
3651                 WARN(1, "bad out_state %d", con->v2.out_state);
3652                 break;
3653         }
3654 }
3655
3656 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3657 {
3658         int remaining;
3659         int resid;
3660
3661         WARN_ON(con_secure(con));
3662         WARN_ON(!data_len(con->in_msg));
3663         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3664         resid = iov_iter_count(&con->v2.in_iter);
3665         WARN_ON(!resid);
3666
3667         remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3668         dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3669              remaining);
3670         con->v2.in_iter.count -= resid;
3671         set_in_skip(con, resid + remaining);
3672         con->v2.in_state = IN_S_FINISH_SKIP;
3673 }
3674
3675 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3676 {
3677         int recved, resid;  /* current piece of data */
3678         int remaining;
3679
3680         WARN_ON(con_secure(con));
3681         WARN_ON(!data_len(con->in_msg));
3682         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3683         resid = iov_iter_count(&con->v2.in_iter);
3684         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3685         recved = con->v2.in_bvec.bv_len - resid;
3686         dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3687
3688         if (recved)
3689                 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3690         WARN_ON(resid > con->v2.in_cursor.total_resid);
3691
3692         remaining = CEPH_EPILOGUE_PLAIN_LEN;
3693         dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3694              con->v2.in_cursor.total_resid, remaining);
3695         con->v2.in_iter.count -= resid;
3696         set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3697         con->v2.in_state = IN_S_FINISH_SKIP;
3698 }
3699
3700 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3701 {
3702         int resid;  /* current enc page (not necessarily data) */
3703
3704         WARN_ON(!con_secure(con));
3705         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3706         resid = iov_iter_count(&con->v2.in_iter);
3707         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3708
3709         dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3710              con->v2.in_enc_resid);
3711         con->v2.in_iter.count -= resid;
3712         set_in_skip(con, resid + con->v2.in_enc_resid);
3713         con->v2.in_state = IN_S_FINISH_SKIP;
3714 }
3715
3716 static void revoke_at_prepare_sparse_data(struct ceph_connection *con)
3717 {
3718         int resid;  /* current piece of data */
3719         int remaining;
3720
3721         WARN_ON(con_secure(con));
3722         WARN_ON(!data_len(con->in_msg));
3723         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3724         resid = iov_iter_count(&con->v2.in_iter);
3725         dout("%s con %p resid %d\n", __func__, con, resid);
3726
3727         remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;
3728         con->v2.in_iter.count -= resid;
3729         set_in_skip(con, resid + remaining);
3730         con->v2.in_state = IN_S_FINISH_SKIP;
3731 }
3732
3733 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3734 {
3735         int resid;
3736
3737         resid = iov_iter_count(&con->v2.in_iter);
3738         WARN_ON(!resid);
3739
3740         dout("%s con %p resid %d\n", __func__, con, resid);
3741         con->v2.in_iter.count -= resid;
3742         set_in_skip(con, resid);
3743         con->v2.in_state = IN_S_FINISH_SKIP;
3744 }
3745
3746 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3747 {
3748         switch (con->v2.in_state) {
3749         case IN_S_PREPARE_SPARSE_DATA:
3750         case IN_S_PREPARE_READ_DATA:
3751                 revoke_at_prepare_read_data(con);
3752                 break;
3753         case IN_S_PREPARE_READ_DATA_CONT:
3754                 revoke_at_prepare_read_data_cont(con);
3755                 break;
3756         case IN_S_PREPARE_READ_ENC_PAGE:
3757                 revoke_at_prepare_read_enc_page(con);
3758                 break;
3759         case IN_S_PREPARE_SPARSE_DATA_CONT:
3760                 revoke_at_prepare_sparse_data(con);
3761                 break;
3762         case IN_S_HANDLE_EPILOGUE:
3763                 revoke_at_handle_epilogue(con);
3764                 break;
3765         default:
3766                 WARN(1, "bad in_state %d", con->v2.in_state);
3767                 break;
3768         }
3769 }
3770
3771 bool ceph_con_v2_opened(struct ceph_connection *con)
3772 {
3773         return con->v2.peer_global_seq;
3774 }
3775
3776 void ceph_con_v2_reset_session(struct ceph_connection *con)
3777 {
3778         con->v2.client_cookie = 0;
3779         con->v2.server_cookie = 0;
3780         con->v2.global_seq = 0;
3781         con->v2.connect_seq = 0;
3782         con->v2.peer_global_seq = 0;
3783 }
3784
3785 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3786 {
3787         iov_iter_truncate(&con->v2.in_iter, 0);
3788         iov_iter_truncate(&con->v2.out_iter, 0);
3789         con->v2.out_zero = 0;
3790
3791         clear_in_sign_kvecs(con);
3792         clear_out_sign_kvecs(con);
3793         free_conn_bufs(con);
3794
3795         if (con->v2.in_enc_pages) {
3796                 WARN_ON(!con->v2.in_enc_page_cnt);
3797                 ceph_release_page_vector(con->v2.in_enc_pages,
3798                                          con->v2.in_enc_page_cnt);
3799                 con->v2.in_enc_pages = NULL;
3800                 con->v2.in_enc_page_cnt = 0;
3801         }
3802         if (con->v2.out_enc_pages) {
3803                 WARN_ON(!con->v2.out_enc_page_cnt);
3804                 ceph_release_page_vector(con->v2.out_enc_pages,
3805                                          con->v2.out_enc_page_cnt);
3806                 con->v2.out_enc_pages = NULL;
3807                 con->v2.out_enc_page_cnt = 0;
3808         }
3809
3810         con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3811         memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3812         memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3813
3814         if (con->v2.hmac_tfm) {
3815                 crypto_free_shash(con->v2.hmac_tfm);
3816                 con->v2.hmac_tfm = NULL;
3817         }
3818         if (con->v2.gcm_req) {
3819                 aead_request_free(con->v2.gcm_req);
3820                 con->v2.gcm_req = NULL;
3821         }
3822         if (con->v2.gcm_tfm) {
3823                 crypto_free_aead(con->v2.gcm_tfm);
3824                 con->v2.gcm_tfm = NULL;
3825         }
3826 }