net/ceph/messenger_v2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Ceph msgr2 protocol implementation
   4  *
   5  * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
   6  */
   7
   8 #include <linux/ceph/ceph_debug.h>
   9
  10 #include <crypto/aead.h>
  11 #include <crypto/algapi.h>  /* for crypto_memneq() */
  12 #include <crypto/hash.h>
  13 #include <crypto/sha2.h>
  14 #include <linux/bvec.h>
  15 #include <linux/crc32c.h>
  16 #include <linux/net.h>
  17 #include <linux/scatterlist.h>
  18 #include <linux/socket.h>
  19 #include <linux/sched/mm.h>
  20 #include <net/sock.h>
  21 #include <net/tcp.h>
  22
  23 #include <linux/ceph/ceph_features.h>
  24 #include <linux/ceph/decode.h>
  25 #include <linux/ceph/libceph.h>
  26 #include <linux/ceph/messenger.h>
  27
  28 #include "crypto.h"  /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
  29
  30 #define FRAME_TAG_HELLO                 1
  31 #define FRAME_TAG_AUTH_REQUEST          2
  32 #define FRAME_TAG_AUTH_BAD_METHOD       3
  33 #define FRAME_TAG_AUTH_REPLY_MORE       4
  34 #define FRAME_TAG_AUTH_REQUEST_MORE     5
  35 #define FRAME_TAG_AUTH_DONE             6
  36 #define FRAME_TAG_AUTH_SIGNATURE        7
  37 #define FRAME_TAG_CLIENT_IDENT          8
  38 #define FRAME_TAG_SERVER_IDENT          9
  39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
  40 #define FRAME_TAG_SESSION_RECONNECT     11
  41 #define FRAME_TAG_SESSION_RESET         12
  42 #define FRAME_TAG_SESSION_RETRY         13
  43 #define FRAME_TAG_SESSION_RETRY_GLOBAL  14
  44 #define FRAME_TAG_SESSION_RECONNECT_OK  15
  45 #define FRAME_TAG_WAIT                  16
  46 #define FRAME_TAG_MESSAGE               17
  47 #define FRAME_TAG_KEEPALIVE2            18
  48 #define FRAME_TAG_KEEPALIVE2_ACK        19
  49 #define FRAME_TAG_ACK                   20
  50
  51 #define FRAME_LATE_STATUS_ABORTED       0x1
  52 #define FRAME_LATE_STATUS_COMPLETE      0xe
  53 #define FRAME_LATE_STATUS_ABORTED_MASK  0xf
  54
  55 #define IN_S_HANDLE_PREAMBLE            1
  56 #define IN_S_HANDLE_CONTROL             2
  57 #define IN_S_HANDLE_CONTROL_REMAINDER   3
  58 #define IN_S_PREPARE_READ_DATA          4
  59 #define IN_S_PREPARE_READ_DATA_CONT     5
  60 #define IN_S_PREPARE_READ_ENC_PAGE      6
  61 #define IN_S_HANDLE_EPILOGUE            7
  62 #define IN_S_FINISH_SKIP                8
  63
  64 #define OUT_S_QUEUE_DATA                1
  65 #define OUT_S_QUEUE_DATA_CONT           2
  66 #define OUT_S_QUEUE_ENC_PAGE            3
  67 #define OUT_S_QUEUE_ZEROS               4
  68 #define OUT_S_FINISH_MESSAGE            5
  69 #define OUT_S_GET_NEXT                  6
  70
  71 #define CTRL_BODY(p)    ((void *)(p) + CEPH_PREAMBLE_LEN)
  72 #define FRONT_PAD(p)    ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
  73 #define MIDDLE_PAD(p)   (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
  74 #define DATA_PAD(p)     (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
  75
  76 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
  77
  78 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
  79 {
  80         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
  81         int ret;
  82
  83         msg.msg_iter = *it;
  84         while (iov_iter_count(it)) {
  85                 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
  86                 if (ret <= 0) {
  87                         if (ret == -EAGAIN)
  88                                 ret = 0;
  89                         return ret;
  90                 }
  91
  92                 iov_iter_advance(it, ret);
  93         }
  94
  95         WARN_ON(msg_data_left(&msg));
  96         return 1;
  97 }
  98
  99 /*
 100  * Read as much as possible.
 101  *
 102  * Return:
 103  *   1 - done, nothing (else) to read
 104  *   0 - socket is empty, need to wait
 105  *  <0 - error
 106  */
 107 static int ceph_tcp_recv(struct ceph_connection *con)
 108 {
 109         int ret;
 110
 111         dout("%s con %p %s %zu\n", __func__, con,
 112              iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
 113              iov_iter_count(&con->v2.in_iter));
 114         ret = do_recvmsg(con->sock, &con->v2.in_iter);
 115         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 116              iov_iter_count(&con->v2.in_iter));
 117         return ret;
 118 }
 119
 120 static int do_sendmsg(struct socket *sock, struct iov_iter *it)
 121 {
 122         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
 123         int ret;
 124
 125         msg.msg_iter = *it;
 126         while (iov_iter_count(it)) {
 127                 ret = sock_sendmsg(sock, &msg);
 128                 if (ret <= 0) {
 129                         if (ret == -EAGAIN)
 130                                 ret = 0;
 131                         return ret;
 132                 }
 133
 134                 iov_iter_advance(it, ret);
 135         }
 136
 137         WARN_ON(msg_data_left(&msg));
 138         return 1;
 139 }
 140
 141 static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
 142 {
 143         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
 144         struct bio_vec bv;
 145         int ret;
 146
 147         if (WARN_ON(!iov_iter_is_bvec(it)))
 148                 return -EINVAL;
 149
 150         while (iov_iter_count(it)) {
 151                 /* iov_iter_iovec() for ITER_BVEC */
 152                 bv.bv_page = it->bvec->bv_page;
 153                 bv.bv_offset = it->bvec->bv_offset + it->iov_offset;
 154                 bv.bv_len = min(iov_iter_count(it),
 155                                 it->bvec->bv_len - it->iov_offset);
 156
 157                 /*
 158                  * sendpage cannot properly handle pages with
 159                  * page_count == 0, we need to fall back to sendmsg if
 160                  * that's the case.
 161                  *
 162                  * Same goes for slab pages: skb_can_coalesce() allows
 163                  * coalescing neighboring slab objects into a single frag
 164                  * which triggers one of hardened usercopy checks.
 165                  */
 166                 if (sendpage_ok(bv.bv_page)) {
 167                         ret = sock->ops->sendpage(sock, bv.bv_page,
 168                                                   bv.bv_offset, bv.bv_len,
 169                                                   CEPH_MSG_FLAGS);
 170                 } else {
 171                         iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len);
 172                         ret = sock_sendmsg(sock, &msg);
 173                 }
 174                 if (ret <= 0) {
 175                         if (ret == -EAGAIN)
 176                                 ret = 0;
 177                         return ret;
 178                 }
 179
 180                 iov_iter_advance(it, ret);
 181         }
 182
 183         return 1;
 184 }
 185
 186 /*
 187  * Write as much as possible.  The socket is expected to be corked,
 188  * so we don't bother with MSG_MORE/MSG_SENDPAGE_NOTLAST here.
 189  *
 190  * Return:
 191  *   1 - done, nothing (else) to write
 192  *   0 - socket is full, need to wait
 193  *  <0 - error
 194  */
 195 static int ceph_tcp_send(struct ceph_connection *con)
 196 {
 197         int ret;
 198
 199         dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
 200              iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
 201         if (con->v2.out_iter_sendpage)
 202                 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
 203         else
 204                 ret = do_sendmsg(con->sock, &con->v2.out_iter);
 205         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 206              iov_iter_count(&con->v2.out_iter));
 207         return ret;
 208 }
 209
 210 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
 211 {
 212         BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
 213         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
 214
 215         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
 216         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
 217         con->v2.in_kvec_cnt++;
 218
 219         con->v2.in_iter.nr_segs++;
 220         con->v2.in_iter.count += len;
 221 }
 222
 223 static void reset_in_kvecs(struct ceph_connection *con)
 224 {
 225         WARN_ON(iov_iter_count(&con->v2.in_iter));
 226
 227         con->v2.in_kvec_cnt = 0;
 228         iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
 229 }
 230
 231 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
 232 {
 233         WARN_ON(iov_iter_count(&con->v2.in_iter));
 234
 235         con->v2.in_bvec = *bv;
 236         iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
 237 }
 238
 239 static void set_in_skip(struct ceph_connection *con, int len)
 240 {
 241         WARN_ON(iov_iter_count(&con->v2.in_iter));
 242
 243         dout("%s con %p len %d\n", __func__, con, len);
 244         iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
 245 }
 246
 247 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
 248 {
 249         BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
 250         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
 251         WARN_ON(con->v2.out_zero);
 252
 253         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
 254         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
 255         con->v2.out_kvec_cnt++;
 256
 257         con->v2.out_iter.nr_segs++;
 258         con->v2.out_iter.count += len;
 259 }
 260
 261 static void reset_out_kvecs(struct ceph_connection *con)
 262 {
 263         WARN_ON(iov_iter_count(&con->v2.out_iter));
 264         WARN_ON(con->v2.out_zero);
 265
 266         con->v2.out_kvec_cnt = 0;
 267
 268         iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
 269         con->v2.out_iter_sendpage = false;
 270 }
 271
 272 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
 273                          bool zerocopy)
 274 {
 275         WARN_ON(iov_iter_count(&con->v2.out_iter));
 276         WARN_ON(con->v2.out_zero);
 277
 278         con->v2.out_bvec = *bv;
 279         con->v2.out_iter_sendpage = zerocopy;
 280         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 281                       con->v2.out_bvec.bv_len);
 282 }
 283
 284 static void set_out_bvec_zero(struct ceph_connection *con)
 285 {
 286         WARN_ON(iov_iter_count(&con->v2.out_iter));
 287         WARN_ON(!con->v2.out_zero);
 288
 289         con->v2.out_bvec.bv_page = ceph_zero_page;
 290         con->v2.out_bvec.bv_offset = 0;
 291         con->v2.out_bvec.bv_len = min(con->v2.out_zero, (int)PAGE_SIZE);
 292         con->v2.out_iter_sendpage = true;
 293         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 294                       con->v2.out_bvec.bv_len);
 295 }
 296
 297 static void out_zero_add(struct ceph_connection *con, int len)
 298 {
 299         dout("%s con %p len %d\n", __func__, con, len);
 300         con->v2.out_zero += len;
 301 }
 302
 303 static void *alloc_conn_buf(struct ceph_connection *con, int len)
 304 {
 305         void *buf;
 306
 307         dout("%s con %p len %d\n", __func__, con, len);
 308
 309         if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
 310                 return NULL;
 311
 312         buf = kvmalloc(len, GFP_NOIO);
 313         if (!buf)
 314                 return NULL;
 315
 316         con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
 317         return buf;
 318 }
 319
 320 static void free_conn_bufs(struct ceph_connection *con)
 321 {
 322         while (con->v2.conn_buf_cnt)
 323                 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
 324 }
 325
 326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
 327 {
 328         BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
 329
 330         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
 331         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
 332         con->v2.in_sign_kvec_cnt++;
 333 }
 334
 335 static void clear_in_sign_kvecs(struct ceph_connection *con)
 336 {
 337         con->v2.in_sign_kvec_cnt = 0;
 338 }
 339
 340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
 341 {
 342         BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
 343
 344         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
 345         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
 346         con->v2.out_sign_kvec_cnt++;
 347 }
 348
 349 static void clear_out_sign_kvecs(struct ceph_connection *con)
 350 {
 351         con->v2.out_sign_kvec_cnt = 0;
 352 }
 353
 354 static bool con_secure(struct ceph_connection *con)
 355 {
 356         return con->v2.con_mode == CEPH_CON_MODE_SECURE;
 357 }
 358
 359 static int front_len(const struct ceph_msg *msg)
 360 {
 361         return le32_to_cpu(msg->hdr.front_len);
 362 }
 363
 364 static int middle_len(const struct ceph_msg *msg)
 365 {
 366         return le32_to_cpu(msg->hdr.middle_len);
 367 }
 368
 369 static int data_len(const struct ceph_msg *msg)
 370 {
 371         return le32_to_cpu(msg->hdr.data_len);
 372 }
 373
 374 static bool need_padding(int len)
 375 {
 376         return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
 377 }
 378
 379 static int padded_len(int len)
 380 {
 381         return ALIGN(len, CEPH_GCM_BLOCK_LEN);
 382 }
 383
 384 static int padding_len(int len)
 385 {
 386         return padded_len(len) - len;
 387 }
 388
 389 /* preamble + control segment */
 390 static int head_onwire_len(int ctrl_len, bool secure)
 391 {
 392         int head_len;
 393         int rem_len;
 394
 395         BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
 396
 397         if (secure) {
 398                 head_len = CEPH_PREAMBLE_SECURE_LEN;
 399                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
 400                         rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
 401                         head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
 402                 }
 403         } else {
 404                 head_len = CEPH_PREAMBLE_PLAIN_LEN;
 405                 if (ctrl_len)
 406                         head_len += ctrl_len + CEPH_CRC_LEN;
 407         }
 408         return head_len;
 409 }
 410
 411 /* front, middle and data segments + epilogue */
 412 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
 413                              bool secure)
 414 {
 415         BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
 416                middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
 417                data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
 418
 419         if (!front_len && !middle_len && !data_len)
 420                 return 0;
 421
 422         if (!secure)
 423                 return front_len + middle_len + data_len +
 424                        CEPH_EPILOGUE_PLAIN_LEN;
 425
 426         return padded_len(front_len) + padded_len(middle_len) +
 427                padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
 428 }
 429
 430 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
 431 {
 432         return __tail_onwire_len(front_len(msg), middle_len(msg),
 433                                  data_len(msg), secure);
 434 }
 435
 436 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
 437 #define MESSAGE_HEAD_PLAIN_LEN  (CEPH_PREAMBLE_PLAIN_LEN +              \
 438                                  sizeof(struct ceph_msg_header2) +      \
 439                                  CEPH_CRC_LEN)
 440
 441 static const int frame_aligns[] = {
 442         sizeof(void *),
 443         sizeof(void *),
 444         sizeof(void *),
 445         PAGE_SIZE
 446 };
 447
 448 /*
 449  * Discards trailing empty segments, unless there is just one segment.
 450  * A frame always has at least one (possibly empty) segment.
 451  */
 452 static int calc_segment_count(const int *lens, int len_cnt)
 453 {
 454         int i;
 455
 456         for (i = len_cnt - 1; i >= 0; i--) {
 457                 if (lens[i])
 458                         return i + 1;
 459         }
 460
 461         return 1;
 462 }
 463
 464 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
 465                             const int *lens, int len_cnt)
 466 {
 467         int i;
 468
 469         memset(desc, 0, sizeof(*desc));
 470
 471         desc->fd_tag = tag;
 472         desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
 473         BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
 474         for (i = 0; i < desc->fd_seg_cnt; i++) {
 475                 desc->fd_lens[i] = lens[i];
 476                 desc->fd_aligns[i] = frame_aligns[i];
 477         }
 478 }
 479
 480 /*
 481  * Preamble crc covers everything up to itself (28 bytes) and
 482  * is calculated and verified irrespective of the connection mode
 483  * (i.e. even if the frame is encrypted).
 484  */
 485 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
 486 {
 487         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 488         void *start = p;
 489         int i;
 490
 491         memset(p, 0, CEPH_PREAMBLE_LEN);
 492
 493         ceph_encode_8(&p, desc->fd_tag);
 494         ceph_encode_8(&p, desc->fd_seg_cnt);
 495         for (i = 0; i < desc->fd_seg_cnt; i++) {
 496                 ceph_encode_32(&p, desc->fd_lens[i]);
 497                 ceph_encode_16(&p, desc->fd_aligns[i]);
 498         }
 499
 500         put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
 501 }
 502
 503 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
 504 {
 505         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 506         u32 crc, expected_crc;
 507         int i;
 508
 509         crc = crc32c(0, p, crcp - p);
 510         expected_crc = get_unaligned_le32(crcp);
 511         if (crc != expected_crc) {
 512                 pr_err("bad preamble crc, calculated %u, expected %u\n",
 513                        crc, expected_crc);
 514                 return -EBADMSG;
 515         }
 516
 517         memset(desc, 0, sizeof(*desc));
 518
 519         desc->fd_tag = ceph_decode_8(&p);
 520         desc->fd_seg_cnt = ceph_decode_8(&p);
 521         if (desc->fd_seg_cnt < 1 ||
 522             desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
 523                 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
 524                 return -EINVAL;
 525         }
 526         for (i = 0; i < desc->fd_seg_cnt; i++) {
 527                 desc->fd_lens[i] = ceph_decode_32(&p);
 528                 desc->fd_aligns[i] = ceph_decode_16(&p);
 529         }
 530
 531         if (desc->fd_lens[0] < 0 ||
 532             desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
 533                 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
 534                 return -EINVAL;
 535         }
 536         if (desc->fd_lens[1] < 0 ||
 537             desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
 538                 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
 539                 return -EINVAL;
 540         }
 541         if (desc->fd_lens[2] < 0 ||
 542             desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
 543                 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
 544                 return -EINVAL;
 545         }
 546         if (desc->fd_lens[3] < 0 ||
 547             desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
 548                 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
 549                 return -EINVAL;
 550         }
 551
 552         /*
 553          * This would fire for FRAME_TAG_WAIT (it has one empty
 554          * segment), but we should never get it as client.
 555          */
 556         if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
 557                 pr_err("last segment empty, segment count %d\n",
 558                        desc->fd_seg_cnt);
 559                 return -EINVAL;
 560         }
 561
 562         return 0;
 563 }
 564
 565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
 566 {
 567         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 568                                                  FRAME_LATE_STATUS_COMPLETE;
 569         cpu_to_le32s(&con->v2.out_epil.front_crc);
 570         cpu_to_le32s(&con->v2.out_epil.middle_crc);
 571         cpu_to_le32s(&con->v2.out_epil.data_crc);
 572 }
 573
 574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
 575 {
 576         memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
 577         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 578                                                  FRAME_LATE_STATUS_COMPLETE;
 579 }
 580
 581 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
 582                            u32 *data_crc)
 583 {
 584         u8 late_status;
 585
 586         late_status = ceph_decode_8(&p);
 587         if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
 588                         FRAME_LATE_STATUS_COMPLETE) {
 589                 /* we should never get an aborted message as client */
 590                 pr_err("bad late_status 0x%x\n", late_status);
 591                 return -EINVAL;
 592         }
 593
 594         if (front_crc && middle_crc && data_crc) {
 595                 *front_crc = ceph_decode_32(&p);
 596                 *middle_crc = ceph_decode_32(&p);
 597                 *data_crc = ceph_decode_32(&p);
 598         }
 599
 600         return 0;
 601 }
 602
 603 static void fill_header(struct ceph_msg_header *hdr,
 604                         const struct ceph_msg_header2 *hdr2,
 605                         int front_len, int middle_len, int data_len,
 606                         const struct ceph_entity_name *peer_name)
 607 {
 608         hdr->seq = hdr2->seq;
 609         hdr->tid = hdr2->tid;
 610         hdr->type = hdr2->type;
 611         hdr->priority = hdr2->priority;
 612         hdr->version = hdr2->version;
 613         hdr->front_len = cpu_to_le32(front_len);
 614         hdr->middle_len = cpu_to_le32(middle_len);
 615         hdr->data_len = cpu_to_le32(data_len);
 616         hdr->data_off = hdr2->data_off;
 617         hdr->src = *peer_name;
 618         hdr->compat_version = hdr2->compat_version;
 619         hdr->reserved = 0;
 620         hdr->crc = 0;
 621 }
 622
 623 static void fill_header2(struct ceph_msg_header2 *hdr2,
 624                          const struct ceph_msg_header *hdr, u64 ack_seq)
 625 {
 626         hdr2->seq = hdr->seq;
 627         hdr2->tid = hdr->tid;
 628         hdr2->type = hdr->type;
 629         hdr2->priority = hdr->priority;
 630         hdr2->version = hdr->version;
 631         hdr2->data_pre_padding_len = 0;
 632         hdr2->data_off = hdr->data_off;
 633         hdr2->ack_seq = cpu_to_le64(ack_seq);
 634         hdr2->flags = 0;
 635         hdr2->compat_version = hdr->compat_version;
 636         hdr2->reserved = 0;
 637 }
 638
 639 static int verify_control_crc(struct ceph_connection *con)
 640 {
 641         int ctrl_len = con->v2.in_desc.fd_lens[0];
 642         u32 crc, expected_crc;
 643
 644         WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
 645         WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
 646
 647         crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
 648         expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
 649         if (crc != expected_crc) {
 650                 pr_err("bad control crc, calculated %u, expected %u\n",
 651                        crc, expected_crc);
 652                 return -EBADMSG;
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
 659                                 u32 middle_crc, u32 data_crc)
 660 {
 661         if (front_len(con->in_msg)) {
 662                 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
 663                                            front_len(con->in_msg));
 664         } else {
 665                 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
 666                 con->in_front_crc = -1;
 667         }
 668
 669         if (middle_len(con->in_msg))
 670                 con->in_middle_crc = crc32c(-1,
 671                                             con->in_msg->middle->vec.iov_base,
 672                                             middle_len(con->in_msg));
 673         else if (data_len(con->in_msg))
 674                 con->in_middle_crc = -1;
 675         else
 676                 con->in_middle_crc = 0;
 677
 678         if (!data_len(con->in_msg))
 679                 con->in_data_crc = 0;
 680
 681         dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
 682              con->in_front_crc, con->in_middle_crc, con->in_data_crc);
 683
 684         if (con->in_front_crc != front_crc) {
 685                 pr_err("bad front crc, calculated %u, expected %u\n",
 686                        con->in_front_crc, front_crc);
 687                 return -EBADMSG;
 688         }
 689         if (con->in_middle_crc != middle_crc) {
 690                 pr_err("bad middle crc, calculated %u, expected %u\n",
 691                        con->in_middle_crc, middle_crc);
 692                 return -EBADMSG;
 693         }
 694         if (con->in_data_crc != data_crc) {
 695                 pr_err("bad data crc, calculated %u, expected %u\n",
 696                        con->in_data_crc, data_crc);
 697                 return -EBADMSG;
 698         }
 699
 700         return 0;
 701 }
 702
 703 static int setup_crypto(struct ceph_connection *con,
 704                         const u8 *session_key, int session_key_len,
 705                         const u8 *con_secret, int con_secret_len)
 706 {
 707         unsigned int noio_flag;
 708         int ret;
 709
 710         dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
 711              __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
 712         WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
 713
 714         if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
 715             con->v2.con_mode != CEPH_CON_MODE_SECURE) {
 716                 pr_err("bad con_mode %d\n", con->v2.con_mode);
 717                 return -EINVAL;
 718         }
 719
 720         if (!session_key_len) {
 721                 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
 722                 WARN_ON(con_secret_len);
 723                 return 0;  /* auth_none */
 724         }
 725
 726         noio_flag = memalloc_noio_save();
 727         con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
 728         memalloc_noio_restore(noio_flag);
 729         if (IS_ERR(con->v2.hmac_tfm)) {
 730                 ret = PTR_ERR(con->v2.hmac_tfm);
 731                 con->v2.hmac_tfm = NULL;
 732                 pr_err("failed to allocate hmac tfm context: %d\n", ret);
 733                 return ret;
 734         }
 735
 736         WARN_ON((unsigned long)session_key &
 737                 crypto_shash_alignmask(con->v2.hmac_tfm));
 738         ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
 739                                   session_key_len);
 740         if (ret) {
 741                 pr_err("failed to set hmac key: %d\n", ret);
 742                 return ret;
 743         }
 744
 745         if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
 746                 WARN_ON(con_secret_len);
 747                 return 0;  /* auth_x, plain mode */
 748         }
 749
 750         if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
 751                 pr_err("con_secret too small %d\n", con_secret_len);
 752                 return -EINVAL;
 753         }
 754
 755         noio_flag = memalloc_noio_save();
 756         con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
 757         memalloc_noio_restore(noio_flag);
 758         if (IS_ERR(con->v2.gcm_tfm)) {
 759                 ret = PTR_ERR(con->v2.gcm_tfm);
 760                 con->v2.gcm_tfm = NULL;
 761                 pr_err("failed to allocate gcm tfm context: %d\n", ret);
 762                 return ret;
 763         }
 764
 765         WARN_ON((unsigned long)con_secret &
 766                 crypto_aead_alignmask(con->v2.gcm_tfm));
 767         ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
 768         if (ret) {
 769                 pr_err("failed to set gcm key: %d\n", ret);
 770                 return ret;
 771         }
 772
 773         WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
 774         ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
 775         if (ret) {
 776                 pr_err("failed to set gcm tag size: %d\n", ret);
 777                 return ret;
 778         }
 779
 780         con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
 781         if (!con->v2.gcm_req) {
 782                 pr_err("failed to allocate gcm request\n");
 783                 return -ENOMEM;
 784         }
 785
 786         crypto_init_wait(&con->v2.gcm_wait);
 787         aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 788                                   crypto_req_done, &con->v2.gcm_wait);
 789
 790         memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
 791                CEPH_GCM_IV_LEN);
 792         memcpy(&con->v2.out_gcm_nonce,
 793                con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
 794                CEPH_GCM_IV_LEN);
 795         return 0;  /* auth_x, secure mode */
 796 }
 797
 798 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
 799                        int kvec_cnt, u8 *hmac)
 800 {
 801         SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm);  /* tfm arg is ignored */
 802         int ret;
 803         int i;
 804
 805         dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
 806              con->v2.hmac_tfm, kvec_cnt);
 807
 808         if (!con->v2.hmac_tfm) {
 809                 memset(hmac, 0, SHA256_DIGEST_SIZE);
 810                 return 0;  /* auth_none */
 811         }
 812
 813         desc->tfm = con->v2.hmac_tfm;
 814         ret = crypto_shash_init(desc);
 815         if (ret)
 816                 goto out;
 817
 818         for (i = 0; i < kvec_cnt; i++) {
 819                 WARN_ON((unsigned long)kvecs[i].iov_base &
 820                         crypto_shash_alignmask(con->v2.hmac_tfm));
 821                 ret = crypto_shash_update(desc, kvecs[i].iov_base,
 822                                           kvecs[i].iov_len);
 823                 if (ret)
 824                         goto out;
 825         }
 826
 827         ret = crypto_shash_final(desc, hmac);
 828
 829 out:
 830         shash_desc_zero(desc);
 831         return ret;  /* auth_x, both plain and secure modes */
 832 }
 833
 834 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
 835 {
 836         u64 counter;
 837
 838         counter = le64_to_cpu(nonce->counter);
 839         nonce->counter = cpu_to_le64(counter + 1);
 840 }
 841
 842 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
 843                      struct scatterlist *src, struct scatterlist *dst,
 844                      int src_len)
 845 {
 846         struct ceph_gcm_nonce *nonce;
 847         int ret;
 848
 849         nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
 850
 851         aead_request_set_ad(con->v2.gcm_req, 0);  /* no AAD */
 852         aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
 853         ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
 854                                         crypto_aead_decrypt(con->v2.gcm_req),
 855                               &con->v2.gcm_wait);
 856         if (ret)
 857                 return ret;
 858
 859         gcm_inc_nonce(nonce);
 860         return 0;
 861 }
 862
 863 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
 864                         struct bio_vec *bv)
 865 {
 866         struct page *page;
 867         size_t off, len;
 868
 869         WARN_ON(!cursor->total_resid);
 870
 871         /* skip zero-length data items */
 872         while (!cursor->resid)
 873                 ceph_msg_data_advance(cursor, 0);
 874
 875         /* get a piece of data, cursor isn't advanced */
 876         page = ceph_msg_data_next(cursor, &off, &len);
 877
 878         bv->bv_page = page;
 879         bv->bv_offset = off;
 880         bv->bv_len = len;
 881 }
 882
 883 static int calc_sg_cnt(void *buf, int buf_len)
 884 {
 885         int sg_cnt;
 886
 887         if (!buf_len)
 888                 return 0;
 889
 890         sg_cnt = need_padding(buf_len) ? 1 : 0;
 891         if (is_vmalloc_addr(buf)) {
 892                 WARN_ON(offset_in_page(buf));
 893                 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
 894         } else {
 895                 sg_cnt++;
 896         }
 897
 898         return sg_cnt;
 899 }
 900
 901 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
 902 {
 903         int data_len = cursor->total_resid;
 904         struct bio_vec bv;
 905         int sg_cnt;
 906
 907         if (!data_len)
 908                 return 0;
 909
 910         sg_cnt = need_padding(data_len) ? 1 : 0;
 911         do {
 912                 get_bvec_at(cursor, &bv);
 913                 sg_cnt++;
 914
 915                 ceph_msg_data_advance(cursor, bv.bv_len);
 916         } while (cursor->total_resid);
 917
 918         return sg_cnt;
 919 }
 920
 921 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
 922 {
 923         void *end = buf + buf_len;
 924         struct page *page;
 925         int len;
 926         void *p;
 927
 928         if (!buf_len)
 929                 return;
 930
 931         if (is_vmalloc_addr(buf)) {
 932                 p = buf;
 933                 do {
 934                         page = vmalloc_to_page(p);
 935                         len = min_t(int, end - p, PAGE_SIZE);
 936                         WARN_ON(!page || !len || offset_in_page(p));
 937                         sg_set_page(*sg, page, len, 0);
 938                         *sg = sg_next(*sg);
 939                         p += len;
 940                 } while (p != end);
 941         } else {
 942                 sg_set_buf(*sg, buf, buf_len);
 943                 *sg = sg_next(*sg);
 944         }
 945
 946         if (need_padding(buf_len)) {
 947                 sg_set_buf(*sg, pad, padding_len(buf_len));
 948                 *sg = sg_next(*sg);
 949         }
 950 }
 951
 952 static void init_sgs_cursor(struct scatterlist **sg,
 953                             struct ceph_msg_data_cursor *cursor, u8 *pad)
 954 {
 955         int data_len = cursor->total_resid;
 956         struct bio_vec bv;
 957
 958         if (!data_len)
 959                 return;
 960
 961         do {
 962                 get_bvec_at(cursor, &bv);
 963                 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
 964                 *sg = sg_next(*sg);
 965
 966                 ceph_msg_data_advance(cursor, bv.bv_len);
 967         } while (cursor->total_resid);
 968
 969         if (need_padding(data_len)) {
 970                 sg_set_buf(*sg, pad, padding_len(data_len));
 971                 *sg = sg_next(*sg);
 972         }
 973 }
 974
 975 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
 976                              u8 *front_pad, u8 *middle_pad, u8 *data_pad,
 977                              void *epilogue, bool add_tag)
 978 {
 979         struct ceph_msg_data_cursor cursor;
 980         struct scatterlist *cur_sg;
 981         int sg_cnt;
 982         int ret;
 983
 984         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
 985                 return 0;
 986
 987         sg_cnt = 1;  /* epilogue + [auth tag] */
 988         if (front_len(msg))
 989                 sg_cnt += calc_sg_cnt(msg->front.iov_base,
 990                                       front_len(msg));
 991         if (middle_len(msg))
 992                 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
 993                                       middle_len(msg));
 994         if (data_len(msg)) {
 995                 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
 996                 sg_cnt += calc_sg_cnt_cursor(&cursor);
 997         }
 998
 999         ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1000         if (ret)
1001                 return ret;
1002
1003         cur_sg = sgt->sgl;
1004         if (front_len(msg))
1005                 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1006                          front_pad);
1007         if (middle_len(msg))
1008                 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1009                          middle_pad);
1010         if (data_len(msg)) {
1011                 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
1012                 init_sgs_cursor(&cur_sg, &cursor, data_pad);
1013         }
1014
1015         WARN_ON(!sg_is_last(cur_sg));
1016         sg_set_buf(cur_sg, epilogue,
1017                    CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1018         return 0;
1019 }
1020
1021 static int decrypt_preamble(struct ceph_connection *con)
1022 {
1023         struct scatterlist sg;
1024
1025         sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1026         return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1027 }
1028
1029 static int decrypt_control_remainder(struct ceph_connection *con)
1030 {
1031         int ctrl_len = con->v2.in_desc.fd_lens[0];
1032         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1033         int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1034         struct scatterlist sgs[2];
1035
1036         WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1037         WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1038
1039         sg_init_table(sgs, 2);
1040         sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1041         sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1042
1043         return gcm_crypt(con, false, sgs, sgs,
1044                          padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1045 }
1046
1047 static int decrypt_tail(struct ceph_connection *con)
1048 {
1049         struct sg_table enc_sgt = {};
1050         struct sg_table sgt = {};
1051         int tail_len;
1052         int ret;
1053
1054         tail_len = tail_onwire_len(con->in_msg, true);
1055         ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
1056                                         con->v2.in_enc_page_cnt, 0, tail_len,
1057                                         GFP_NOIO);
1058         if (ret)
1059                 goto out;
1060
1061         ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1062                         MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1063                         con->v2.in_buf, true);
1064         if (ret)
1065                 goto out;
1066
1067         dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1068              con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1069         ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1070         if (ret)
1071                 goto out;
1072
1073         WARN_ON(!con->v2.in_enc_page_cnt);
1074         ceph_release_page_vector(con->v2.in_enc_pages,
1075                                  con->v2.in_enc_page_cnt);
1076         con->v2.in_enc_pages = NULL;
1077         con->v2.in_enc_page_cnt = 0;
1078
1079 out:
1080         sg_free_table(&sgt);
1081         sg_free_table(&enc_sgt);
1082         return ret;
1083 }
1084
1085 static int prepare_banner(struct ceph_connection *con)
1086 {
1087         int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1088         void *buf, *p;
1089
1090         buf = alloc_conn_buf(con, buf_len);
1091         if (!buf)
1092                 return -ENOMEM;
1093
1094         p = buf;
1095         ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1096         ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1097         ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1098         ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1099         WARN_ON(p != buf + buf_len);
1100
1101         add_out_kvec(con, buf, buf_len);
1102         add_out_sign_kvec(con, buf, buf_len);
1103         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1104         return 0;
1105 }
1106
1107 /*
1108  * base:
1109  *   preamble
1110  *   control body (ctrl_len bytes)
1111  *   space for control crc
1112  *
1113  * extdata (optional):
1114  *   control body (extdata_len bytes)
1115  *
1116  * Compute control crc and gather base and extdata into:
1117  *
1118  *   preamble
1119  *   control body (ctrl_len + extdata_len bytes)
1120  *   control crc
1121  *
1122  * Preamble should already be encoded at the start of base.
1123  */
1124 static void prepare_head_plain(struct ceph_connection *con, void *base,
1125                                int ctrl_len, void *extdata, int extdata_len,
1126                                bool to_be_signed)
1127 {
1128         int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1129         void *crcp = base + base_len - CEPH_CRC_LEN;
1130         u32 crc;
1131
1132         crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1133         if (extdata_len)
1134                 crc = crc32c(crc, extdata, extdata_len);
1135         put_unaligned_le32(crc, crcp);
1136
1137         if (!extdata_len) {
1138                 add_out_kvec(con, base, base_len);
1139                 if (to_be_signed)
1140                         add_out_sign_kvec(con, base, base_len);
1141                 return;
1142         }
1143
1144         add_out_kvec(con, base, crcp - base);
1145         add_out_kvec(con, extdata, extdata_len);
1146         add_out_kvec(con, crcp, CEPH_CRC_LEN);
1147         if (to_be_signed) {
1148                 add_out_sign_kvec(con, base, crcp - base);
1149                 add_out_sign_kvec(con, extdata, extdata_len);
1150                 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1151         }
1152 }
1153
1154 static int prepare_head_secure_small(struct ceph_connection *con,
1155                                      void *base, int ctrl_len)
1156 {
1157         struct scatterlist sg;
1158         int ret;
1159
1160         /* inline buffer padding? */
1161         if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1162                 memset(CTRL_BODY(base) + ctrl_len, 0,
1163                        CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1164
1165         sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1166         ret = gcm_crypt(con, true, &sg, &sg,
1167                         CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1168         if (ret)
1169                 return ret;
1170
1171         add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1172         return 0;
1173 }
1174
1175 /*
1176  * base:
1177  *   preamble
1178  *   control body (ctrl_len bytes)
1179  *   space for padding, if needed
1180  *   space for control remainder auth tag
1181  *   space for preamble auth tag
1182  *
1183  * Encrypt preamble and the inline portion, then encrypt the remainder
1184  * and gather into:
1185  *
1186  *   preamble
1187  *   control body (48 bytes)
1188  *   preamble auth tag
1189  *   control body (ctrl_len - 48 bytes)
1190  *   zero padding, if needed
1191  *   control remainder auth tag
1192  *
1193  * Preamble should already be encoded at the start of base.
1194  */
1195 static int prepare_head_secure_big(struct ceph_connection *con,
1196                                    void *base, int ctrl_len)
1197 {
1198         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1199         void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1200         void *rem_tag = rem + padded_len(rem_len);
1201         void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1202         struct scatterlist sgs[2];
1203         int ret;
1204
1205         sg_init_table(sgs, 2);
1206         sg_set_buf(&sgs[0], base, rem - base);
1207         sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1208         ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1209         if (ret)
1210                 return ret;
1211
1212         /* control remainder padding? */
1213         if (need_padding(rem_len))
1214                 memset(rem + rem_len, 0, padding_len(rem_len));
1215
1216         sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1217         ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1218         if (ret)
1219                 return ret;
1220
1221         add_out_kvec(con, base, rem - base);
1222         add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1223         add_out_kvec(con, rem, pmbl_tag - rem);
1224         return 0;
1225 }
1226
1227 static int __prepare_control(struct ceph_connection *con, int tag,
1228                              void *base, int ctrl_len, void *extdata,
1229                              int extdata_len, bool to_be_signed)
1230 {
1231         int total_len = ctrl_len + extdata_len;
1232         struct ceph_frame_desc desc;
1233         int ret;
1234
1235         dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1236              total_len, ctrl_len, extdata_len);
1237
1238         /* extdata may be vmalloc'ed but not base */
1239         if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1240                 return -EINVAL;
1241
1242         init_frame_desc(&desc, tag, &total_len, 1);
1243         encode_preamble(&desc, base);
1244
1245         if (con_secure(con)) {
1246                 if (WARN_ON(extdata_len || to_be_signed))
1247                         return -EINVAL;
1248
1249                 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1250                         /* fully inlined, inline buffer may need padding */
1251                         ret = prepare_head_secure_small(con, base, ctrl_len);
1252                 else
1253                         /* partially inlined, inline buffer is full */
1254                         ret = prepare_head_secure_big(con, base, ctrl_len);
1255                 if (ret)
1256                         return ret;
1257         } else {
1258                 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1259                                    to_be_signed);
1260         }
1261
1262         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1263         return 0;
1264 }
1265
1266 static int prepare_control(struct ceph_connection *con, int tag,
1267                            void *base, int ctrl_len)
1268 {
1269         return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1270 }
1271
1272 static int prepare_hello(struct ceph_connection *con)
1273 {
1274         void *buf, *p;
1275         int ctrl_len;
1276
1277         ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1278         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1279         if (!buf)
1280                 return -ENOMEM;
1281
1282         p = CTRL_BODY(buf);
1283         ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1284         ceph_encode_entity_addr(&p, &con->peer_addr);
1285         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1286
1287         return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1288                                  NULL, 0, true);
1289 }
1290
1291 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1292 #define AUTH_BUF_LEN    (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1293
1294 static int prepare_auth_request(struct ceph_connection *con)
1295 {
1296         void *authorizer, *authorizer_copy;
1297         int ctrl_len, authorizer_len;
1298         void *buf;
1299         int ret;
1300
1301         ctrl_len = AUTH_BUF_LEN;
1302         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1303         if (!buf)
1304                 return -ENOMEM;
1305
1306         mutex_unlock(&con->mutex);
1307         ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1308                                          &authorizer, &authorizer_len);
1309         mutex_lock(&con->mutex);
1310         if (con->state != CEPH_CON_S_V2_HELLO) {
1311                 dout("%s con %p state changed to %d\n", __func__, con,
1312                      con->state);
1313                 return -EAGAIN;
1314         }
1315
1316         dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1317         if (ret)
1318                 return ret;
1319
1320         authorizer_copy = alloc_conn_buf(con, authorizer_len);
1321         if (!authorizer_copy)
1322                 return -ENOMEM;
1323
1324         memcpy(authorizer_copy, authorizer, authorizer_len);
1325
1326         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1327                                  authorizer_copy, authorizer_len, true);
1328 }
1329
1330 static int prepare_auth_request_more(struct ceph_connection *con,
1331                                      void *reply, int reply_len)
1332 {
1333         int ctrl_len, authorizer_len;
1334         void *authorizer;
1335         void *buf;
1336         int ret;
1337
1338         ctrl_len = AUTH_BUF_LEN;
1339         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1340         if (!buf)
1341                 return -ENOMEM;
1342
1343         mutex_unlock(&con->mutex);
1344         ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1345                                                CTRL_BODY(buf), &ctrl_len,
1346                                                &authorizer, &authorizer_len);
1347         mutex_lock(&con->mutex);
1348         if (con->state != CEPH_CON_S_V2_AUTH) {
1349                 dout("%s con %p state changed to %d\n", __func__, con,
1350                      con->state);
1351                 return -EAGAIN;
1352         }
1353
1354         dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1355         if (ret)
1356                 return ret;
1357
1358         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1359                                  ctrl_len, authorizer, authorizer_len, true);
1360 }
1361
1362 static int prepare_auth_signature(struct ceph_connection *con)
1363 {
1364         void *buf;
1365         int ret;
1366
1367         buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1368                                                   con_secure(con)));
1369         if (!buf)
1370                 return -ENOMEM;
1371
1372         ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1373                           CTRL_BODY(buf));
1374         if (ret)
1375                 return ret;
1376
1377         return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1378                                SHA256_DIGEST_SIZE);
1379 }
1380
1381 static int prepare_client_ident(struct ceph_connection *con)
1382 {
1383         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1384         struct ceph_client *client = from_msgr(con->msgr);
1385         u64 global_id = ceph_client_gid(client);
1386         void *buf, *p;
1387         int ctrl_len;
1388
1389         WARN_ON(con->v2.server_cookie);
1390         WARN_ON(con->v2.connect_seq);
1391         WARN_ON(con->v2.peer_global_seq);
1392
1393         if (!con->v2.client_cookie) {
1394                 do {
1395                         get_random_bytes(&con->v2.client_cookie,
1396                                          sizeof(con->v2.client_cookie));
1397                 } while (!con->v2.client_cookie);
1398                 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1399                      con->v2.client_cookie);
1400         } else {
1401                 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1402                      con->v2.client_cookie);
1403         }
1404
1405         dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1406              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1407              ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1408              global_id, con->v2.global_seq, client->supported_features,
1409              client->required_features, con->v2.client_cookie);
1410
1411         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1412                    ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1413         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1414         if (!buf)
1415                 return -ENOMEM;
1416
1417         p = CTRL_BODY(buf);
1418         ceph_encode_8(&p, 2);  /* addrvec marker */
1419         ceph_encode_32(&p, 1);  /* addr_cnt */
1420         ceph_encode_entity_addr(&p, my_addr);
1421         ceph_encode_entity_addr(&p, &con->peer_addr);
1422         ceph_encode_64(&p, global_id);
1423         ceph_encode_64(&p, con->v2.global_seq);
1424         ceph_encode_64(&p, client->supported_features);
1425         ceph_encode_64(&p, client->required_features);
1426         ceph_encode_64(&p, 0);  /* flags */
1427         ceph_encode_64(&p, con->v2.client_cookie);
1428         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1429
1430         return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1431 }
1432
1433 static int prepare_session_reconnect(struct ceph_connection *con)
1434 {
1435         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1436         void *buf, *p;
1437         int ctrl_len;
1438
1439         WARN_ON(!con->v2.client_cookie);
1440         WARN_ON(!con->v2.server_cookie);
1441         WARN_ON(!con->v2.connect_seq);
1442         WARN_ON(!con->v2.peer_global_seq);
1443
1444         dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1445              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1446              con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1447              con->v2.connect_seq, con->in_seq);
1448
1449         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1450         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1451         if (!buf)
1452                 return -ENOMEM;
1453
1454         p = CTRL_BODY(buf);
1455         ceph_encode_8(&p, 2);  /* entity_addrvec_t marker */
1456         ceph_encode_32(&p, 1);  /* my_addrs len */
1457         ceph_encode_entity_addr(&p, my_addr);
1458         ceph_encode_64(&p, con->v2.client_cookie);
1459         ceph_encode_64(&p, con->v2.server_cookie);
1460         ceph_encode_64(&p, con->v2.global_seq);
1461         ceph_encode_64(&p, con->v2.connect_seq);
1462         ceph_encode_64(&p, con->in_seq);
1463         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1464
1465         return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1466 }
1467
1468 static int prepare_keepalive2(struct ceph_connection *con)
1469 {
1470         struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1471         struct timespec64 now;
1472
1473         ktime_get_real_ts64(&now);
1474         dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1475              now.tv_nsec);
1476
1477         ceph_encode_timespec64(ts, &now);
1478
1479         reset_out_kvecs(con);
1480         return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1481                                sizeof(struct ceph_timespec));
1482 }
1483
1484 static int prepare_ack(struct ceph_connection *con)
1485 {
1486         void *p;
1487
1488         dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1489              con->in_seq_acked, con->in_seq);
1490         con->in_seq_acked = con->in_seq;
1491
1492         p = CTRL_BODY(con->v2.out_buf);
1493         ceph_encode_64(&p, con->in_seq_acked);
1494
1495         reset_out_kvecs(con);
1496         return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1497 }
1498
1499 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1500 {
1501         dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1502              con->out_msg, aborted, con->v2.out_epil.front_crc,
1503              con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1504
1505         encode_epilogue_plain(con, aborted);
1506         add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1507 }
1508
1509 /*
1510  * For "used" empty segments, crc is -1.  For unused (trailing)
1511  * segments, crc is 0.
1512  */
1513 static void prepare_message_plain(struct ceph_connection *con)
1514 {
1515         struct ceph_msg *msg = con->out_msg;
1516
1517         prepare_head_plain(con, con->v2.out_buf,
1518                            sizeof(struct ceph_msg_header2), NULL, 0, false);
1519
1520         if (!front_len(msg) && !middle_len(msg)) {
1521                 if (!data_len(msg)) {
1522                         /*
1523                          * Empty message: once the head is written,
1524                          * we are done -- there is no epilogue.
1525                          */
1526                         con->v2.out_state = OUT_S_FINISH_MESSAGE;
1527                         return;
1528                 }
1529
1530                 con->v2.out_epil.front_crc = -1;
1531                 con->v2.out_epil.middle_crc = -1;
1532                 con->v2.out_state = OUT_S_QUEUE_DATA;
1533                 return;
1534         }
1535
1536         if (front_len(msg)) {
1537                 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1538                                                     front_len(msg));
1539                 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1540         } else {
1541                 /* middle (at least) is there, checked above */
1542                 con->v2.out_epil.front_crc = -1;
1543         }
1544
1545         if (middle_len(msg)) {
1546                 con->v2.out_epil.middle_crc =
1547                         crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1548                 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1549         } else {
1550                 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1551         }
1552
1553         if (data_len(msg)) {
1554                 con->v2.out_state = OUT_S_QUEUE_DATA;
1555         } else {
1556                 con->v2.out_epil.data_crc = 0;
1557                 prepare_epilogue_plain(con, false);
1558                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1559         }
1560 }
1561
1562 /*
1563  * Unfortunately the kernel crypto API doesn't support streaming
1564  * (piecewise) operation for AEAD algorithms, so we can't get away
1565  * with a fixed size buffer and a couple sgs.  Instead, we have to
1566  * allocate pages for the entire tail of the message (currently up
1567  * to ~32M) and two sgs arrays (up to ~256K each)...
1568  */
1569 static int prepare_message_secure(struct ceph_connection *con)
1570 {
1571         void *zerop = page_address(ceph_zero_page);
1572         struct sg_table enc_sgt = {};
1573         struct sg_table sgt = {};
1574         struct page **enc_pages;
1575         int enc_page_cnt;
1576         int tail_len;
1577         int ret;
1578
1579         ret = prepare_head_secure_small(con, con->v2.out_buf,
1580                                         sizeof(struct ceph_msg_header2));
1581         if (ret)
1582                 return ret;
1583
1584         tail_len = tail_onwire_len(con->out_msg, true);
1585         if (!tail_len) {
1586                 /*
1587                  * Empty message: once the head is written,
1588                  * we are done -- there is no epilogue.
1589                  */
1590                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1591                 return 0;
1592         }
1593
1594         encode_epilogue_secure(con, false);
1595         ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1596                                 &con->v2.out_epil, false);
1597         if (ret)
1598                 goto out;
1599
1600         enc_page_cnt = calc_pages_for(0, tail_len);
1601         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1602         if (IS_ERR(enc_pages)) {
1603                 ret = PTR_ERR(enc_pages);
1604                 goto out;
1605         }
1606
1607         WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1608         con->v2.out_enc_pages = enc_pages;
1609         con->v2.out_enc_page_cnt = enc_page_cnt;
1610         con->v2.out_enc_resid = tail_len;
1611         con->v2.out_enc_i = 0;
1612
1613         ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1614                                         0, tail_len, GFP_NOIO);
1615         if (ret)
1616                 goto out;
1617
1618         ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1619                         tail_len - CEPH_GCM_TAG_LEN);
1620         if (ret)
1621                 goto out;
1622
1623         dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1624              con->out_msg, sgt.orig_nents, enc_page_cnt);
1625         con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1626
1627 out:
1628         sg_free_table(&sgt);
1629         sg_free_table(&enc_sgt);
1630         return ret;
1631 }
1632
1633 static int prepare_message(struct ceph_connection *con)
1634 {
1635         int lens[] = {
1636                 sizeof(struct ceph_msg_header2),
1637                 front_len(con->out_msg),
1638                 middle_len(con->out_msg),
1639                 data_len(con->out_msg)
1640         };
1641         struct ceph_frame_desc desc;
1642         int ret;
1643
1644         dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1645              con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1646
1647         if (con->in_seq > con->in_seq_acked) {
1648                 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1649                      con->in_seq_acked, con->in_seq);
1650                 con->in_seq_acked = con->in_seq;
1651         }
1652
1653         reset_out_kvecs(con);
1654         init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1655         encode_preamble(&desc, con->v2.out_buf);
1656         fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1657                      con->in_seq_acked);
1658
1659         if (con_secure(con)) {
1660                 ret = prepare_message_secure(con);
1661                 if (ret)
1662                         return ret;
1663         } else {
1664                 prepare_message_plain(con);
1665         }
1666
1667         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1668         return 0;
1669 }
1670
1671 static int prepare_read_banner_prefix(struct ceph_connection *con)
1672 {
1673         void *buf;
1674
1675         buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1676         if (!buf)
1677                 return -ENOMEM;
1678
1679         reset_in_kvecs(con);
1680         add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1681         add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1682         con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1683         return 0;
1684 }
1685
1686 static int prepare_read_banner_payload(struct ceph_connection *con,
1687                                        int payload_len)
1688 {
1689         void *buf;
1690
1691         buf = alloc_conn_buf(con, payload_len);
1692         if (!buf)
1693                 return -ENOMEM;
1694
1695         reset_in_kvecs(con);
1696         add_in_kvec(con, buf, payload_len);
1697         add_in_sign_kvec(con, buf, payload_len);
1698         con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1699         return 0;
1700 }
1701
1702 static void prepare_read_preamble(struct ceph_connection *con)
1703 {
1704         reset_in_kvecs(con);
1705         add_in_kvec(con, con->v2.in_buf,
1706                     con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1707                                       CEPH_PREAMBLE_PLAIN_LEN);
1708         con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1709 }
1710
1711 static int prepare_read_control(struct ceph_connection *con)
1712 {
1713         int ctrl_len = con->v2.in_desc.fd_lens[0];
1714         int head_len;
1715         void *buf;
1716
1717         reset_in_kvecs(con);
1718         if (con->state == CEPH_CON_S_V2_HELLO ||
1719             con->state == CEPH_CON_S_V2_AUTH) {
1720                 head_len = head_onwire_len(ctrl_len, false);
1721                 buf = alloc_conn_buf(con, head_len);
1722                 if (!buf)
1723                         return -ENOMEM;
1724
1725                 /* preserve preamble */
1726                 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1727
1728                 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1729                 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1730                 add_in_sign_kvec(con, buf, head_len);
1731         } else {
1732                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1733                         buf = alloc_conn_buf(con, ctrl_len);
1734                         if (!buf)
1735                                 return -ENOMEM;
1736
1737                         add_in_kvec(con, buf, ctrl_len);
1738                 } else {
1739                         add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1740                 }
1741                 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1742         }
1743         con->v2.in_state = IN_S_HANDLE_CONTROL;
1744         return 0;
1745 }
1746
1747 static int prepare_read_control_remainder(struct ceph_connection *con)
1748 {
1749         int ctrl_len = con->v2.in_desc.fd_lens[0];
1750         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1751         void *buf;
1752
1753         buf = alloc_conn_buf(con, ctrl_len);
1754         if (!buf)
1755                 return -ENOMEM;
1756
1757         memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1758
1759         reset_in_kvecs(con);
1760         add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1761         add_in_kvec(con, con->v2.in_buf,
1762                     padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1763         con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1764         return 0;
1765 }
1766
1767 static int prepare_read_data(struct ceph_connection *con)
1768 {
1769         struct bio_vec bv;
1770
1771         con->in_data_crc = -1;
1772         ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1773                                   data_len(con->in_msg));
1774
1775         get_bvec_at(&con->v2.in_cursor, &bv);
1776         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1777                 if (unlikely(!con->bounce_page)) {
1778                         con->bounce_page = alloc_page(GFP_NOIO);
1779                         if (!con->bounce_page) {
1780                                 pr_err("failed to allocate bounce page\n");
1781                                 return -ENOMEM;
1782                         }
1783                 }
1784
1785                 bv.bv_page = con->bounce_page;
1786                 bv.bv_offset = 0;
1787         }
1788         set_in_bvec(con, &bv);
1789         con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1790         return 0;
1791 }
1792
1793 static void prepare_read_data_cont(struct ceph_connection *con)
1794 {
1795         struct bio_vec bv;
1796
1797         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1798                 con->in_data_crc = crc32c(con->in_data_crc,
1799                                           page_address(con->bounce_page),
1800                                           con->v2.in_bvec.bv_len);
1801
1802                 get_bvec_at(&con->v2.in_cursor, &bv);
1803                 memcpy_to_page(bv.bv_page, bv.bv_offset,
1804                                page_address(con->bounce_page),
1805                                con->v2.in_bvec.bv_len);
1806         } else {
1807                 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1808                                                     con->v2.in_bvec.bv_page,
1809                                                     con->v2.in_bvec.bv_offset,
1810                                                     con->v2.in_bvec.bv_len);
1811         }
1812
1813         ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1814         if (con->v2.in_cursor.total_resid) {
1815                 get_bvec_at(&con->v2.in_cursor, &bv);
1816                 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1817                         bv.bv_page = con->bounce_page;
1818                         bv.bv_offset = 0;
1819                 }
1820                 set_in_bvec(con, &bv);
1821                 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1822                 return;
1823         }
1824
1825         /*
1826          * We've read all data.  Prepare to read epilogue.
1827          */
1828         reset_in_kvecs(con);
1829         add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1830         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1831 }
1832
1833 static int prepare_read_tail_plain(struct ceph_connection *con)
1834 {
1835         struct ceph_msg *msg = con->in_msg;
1836
1837         if (!front_len(msg) && !middle_len(msg)) {
1838                 WARN_ON(!data_len(msg));
1839                 return prepare_read_data(con);
1840         }
1841
1842         reset_in_kvecs(con);
1843         if (front_len(msg)) {
1844                 add_in_kvec(con, msg->front.iov_base, front_len(msg));
1845                 WARN_ON(msg->front.iov_len != front_len(msg));
1846         }
1847         if (middle_len(msg)) {
1848                 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1849                 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
1850         }
1851
1852         if (data_len(msg)) {
1853                 con->v2.in_state = IN_S_PREPARE_READ_DATA;
1854         } else {
1855                 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1856                 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1857         }
1858         return 0;
1859 }
1860
1861 static void prepare_read_enc_page(struct ceph_connection *con)
1862 {
1863         struct bio_vec bv;
1864
1865         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
1866              con->v2.in_enc_resid);
1867         WARN_ON(!con->v2.in_enc_resid);
1868
1869         bv.bv_page = con->v2.in_enc_pages[con->v2.in_enc_i];
1870         bv.bv_offset = 0;
1871         bv.bv_len = min(con->v2.in_enc_resid, (int)PAGE_SIZE);
1872
1873         set_in_bvec(con, &bv);
1874         con->v2.in_enc_i++;
1875         con->v2.in_enc_resid -= bv.bv_len;
1876
1877         if (con->v2.in_enc_resid) {
1878                 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
1879                 return;
1880         }
1881
1882         /*
1883          * We are set to read the last piece of ciphertext (ending
1884          * with epilogue) + auth tag.
1885          */
1886         WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
1887         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1888 }
1889
1890 static int prepare_read_tail_secure(struct ceph_connection *con)
1891 {
1892         struct page **enc_pages;
1893         int enc_page_cnt;
1894         int tail_len;
1895
1896         tail_len = tail_onwire_len(con->in_msg, true);
1897         WARN_ON(!tail_len);
1898
1899         enc_page_cnt = calc_pages_for(0, tail_len);
1900         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1901         if (IS_ERR(enc_pages))
1902                 return PTR_ERR(enc_pages);
1903
1904         WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
1905         con->v2.in_enc_pages = enc_pages;
1906         con->v2.in_enc_page_cnt = enc_page_cnt;
1907         con->v2.in_enc_resid = tail_len;
1908         con->v2.in_enc_i = 0;
1909
1910         prepare_read_enc_page(con);
1911         return 0;
1912 }
1913
1914 static void __finish_skip(struct ceph_connection *con)
1915 {
1916         con->in_seq++;
1917         prepare_read_preamble(con);
1918 }
1919
1920 static void prepare_skip_message(struct ceph_connection *con)
1921 {
1922         struct ceph_frame_desc *desc = &con->v2.in_desc;
1923         int tail_len;
1924
1925         dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
1926              desc->fd_lens[2], desc->fd_lens[3]);
1927
1928         tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
1929                                      desc->fd_lens[3], con_secure(con));
1930         if (!tail_len) {
1931                 __finish_skip(con);
1932         } else {
1933                 set_in_skip(con, tail_len);
1934                 con->v2.in_state = IN_S_FINISH_SKIP;
1935         }
1936 }
1937
1938 static int process_banner_prefix(struct ceph_connection *con)
1939 {
1940         int payload_len;
1941         void *p;
1942
1943         WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
1944
1945         p = con->v2.in_kvecs[0].iov_base;
1946         if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
1947                 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
1948                         con->error_msg = "server is speaking msgr1 protocol";
1949                 else
1950                         con->error_msg = "protocol error, bad banner";
1951                 return -EINVAL;
1952         }
1953
1954         p += CEPH_BANNER_V2_LEN;
1955         payload_len = ceph_decode_16(&p);
1956         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
1957
1958         return prepare_read_banner_payload(con, payload_len);
1959 }
1960
1961 static int process_banner_payload(struct ceph_connection *con)
1962 {
1963         void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
1964         u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
1965         u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
1966         u64 server_feat, server_req_feat;
1967         void *p;
1968         int ret;
1969
1970         p = con->v2.in_kvecs[0].iov_base;
1971         ceph_decode_64_safe(&p, end, server_feat, bad);
1972         ceph_decode_64_safe(&p, end, server_req_feat, bad);
1973
1974         dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
1975              __func__, con, server_feat, server_req_feat);
1976
1977         if (req_feat & ~server_feat) {
1978                 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
1979                        server_feat, req_feat & ~server_feat);
1980                 con->error_msg = "missing required protocol features";
1981                 return -EINVAL;
1982         }
1983         if (server_req_feat & ~feat) {
1984                 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
1985                        feat, server_req_feat & ~feat);
1986                 con->error_msg = "missing required protocol features";
1987                 return -EINVAL;
1988         }
1989
1990         /* no reset_out_kvecs() as our banner may still be pending */
1991         ret = prepare_hello(con);
1992         if (ret) {
1993                 pr_err("prepare_hello failed: %d\n", ret);
1994                 return ret;
1995         }
1996
1997         con->state = CEPH_CON_S_V2_HELLO;
1998         prepare_read_preamble(con);
1999         return 0;
2000
2001 bad:
2002         pr_err("failed to decode banner payload\n");
2003         return -EINVAL;
2004 }
2005
2006 static int process_hello(struct ceph_connection *con, void *p, void *end)
2007 {
2008         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2009         struct ceph_entity_addr addr_for_me;
2010         u8 entity_type;
2011         int ret;
2012
2013         if (con->state != CEPH_CON_S_V2_HELLO) {
2014                 con->error_msg = "protocol error, unexpected hello";
2015                 return -EINVAL;
2016         }
2017
2018         ceph_decode_8_safe(&p, end, entity_type, bad);
2019         ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
2020         if (ret) {
2021                 pr_err("failed to decode addr_for_me: %d\n", ret);
2022                 return ret;
2023         }
2024
2025         dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2026              entity_type, ceph_pr_addr(&addr_for_me));
2027
2028         if (entity_type != con->peer_name.type) {
2029                 pr_err("bad peer type, want %d, got %d\n",
2030                        con->peer_name.type, entity_type);
2031                 con->error_msg = "wrong peer at address";
2032                 return -EINVAL;
2033         }
2034
2035         /*
2036          * Set our address to the address our first peer (i.e. monitor)
2037          * sees that we are connecting from.  If we are behind some sort
2038          * of NAT and want to be identified by some private (not NATed)
2039          * address, ip option should be used.
2040          */
2041         if (ceph_addr_is_blank(my_addr)) {
2042                 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2043                        sizeof(my_addr->in_addr));
2044                 ceph_addr_set_port(my_addr, 0);
2045                 dout("%s con %p set my addr %s, as seen by peer %s\n",
2046                      __func__, con, ceph_pr_addr(my_addr),
2047                      ceph_pr_addr(&con->peer_addr));
2048         } else {
2049                 dout("%s con %p my addr already set %s\n",
2050                      __func__, con, ceph_pr_addr(my_addr));
2051         }
2052
2053         WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
2054         WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2055         WARN_ON(!my_addr->nonce);
2056
2057         /* no reset_out_kvecs() as our hello may still be pending */
2058         ret = prepare_auth_request(con);
2059         if (ret) {
2060                 if (ret != -EAGAIN)
2061                         pr_err("prepare_auth_request failed: %d\n", ret);
2062                 return ret;
2063         }
2064
2065         con->state = CEPH_CON_S_V2_AUTH;
2066         return 0;
2067
2068 bad:
2069         pr_err("failed to decode hello\n");
2070         return -EINVAL;
2071 }
2072
2073 static int process_auth_bad_method(struct ceph_connection *con,
2074                                    void *p, void *end)
2075 {
2076         int allowed_protos[8], allowed_modes[8];
2077         int allowed_proto_cnt, allowed_mode_cnt;
2078         int used_proto, result;
2079         int ret;
2080         int i;
2081
2082         if (con->state != CEPH_CON_S_V2_AUTH) {
2083                 con->error_msg = "protocol error, unexpected auth_bad_method";
2084                 return -EINVAL;
2085         }
2086
2087         ceph_decode_32_safe(&p, end, used_proto, bad);
2088         ceph_decode_32_safe(&p, end, result, bad);
2089         dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2090              result);
2091
2092         ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2093         if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2094                 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2095                 return -EINVAL;
2096         }
2097         for (i = 0; i < allowed_proto_cnt; i++) {
2098                 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2099                 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2100                      i, allowed_protos[i]);
2101         }
2102
2103         ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2104         if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2105                 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2106                 return -EINVAL;
2107         }
2108         for (i = 0; i < allowed_mode_cnt; i++) {
2109                 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2110                 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2111                      i, allowed_modes[i]);
2112         }
2113
2114         mutex_unlock(&con->mutex);
2115         ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2116                                                allowed_protos,
2117                                                allowed_proto_cnt,
2118                                                allowed_modes,
2119                                                allowed_mode_cnt);
2120         mutex_lock(&con->mutex);
2121         if (con->state != CEPH_CON_S_V2_AUTH) {
2122                 dout("%s con %p state changed to %d\n", __func__, con,
2123                      con->state);
2124                 return -EAGAIN;
2125         }
2126
2127         dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2128         return ret;
2129
2130 bad:
2131         pr_err("failed to decode auth_bad_method\n");
2132         return -EINVAL;
2133 }
2134
2135 static int process_auth_reply_more(struct ceph_connection *con,
2136                                    void *p, void *end)
2137 {
2138         int payload_len;
2139         int ret;
2140
2141         if (con->state != CEPH_CON_S_V2_AUTH) {
2142                 con->error_msg = "protocol error, unexpected auth_reply_more";
2143                 return -EINVAL;
2144         }
2145
2146         ceph_decode_32_safe(&p, end, payload_len, bad);
2147         ceph_decode_need(&p, end, payload_len, bad);
2148
2149         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2150
2151         reset_out_kvecs(con);
2152         ret = prepare_auth_request_more(con, p, payload_len);
2153         if (ret) {
2154                 if (ret != -EAGAIN)
2155                         pr_err("prepare_auth_request_more failed: %d\n", ret);
2156                 return ret;
2157         }
2158
2159         return 0;
2160
2161 bad:
2162         pr_err("failed to decode auth_reply_more\n");
2163         return -EINVAL;
2164 }
2165
2166 /*
2167  * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2168  * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2169  * setup_crypto().  __aligned(16) isn't guaranteed to work for stack
2170  * objects, so do it by hand.
2171  */
2172 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2173 {
2174         u8 session_key_buf[CEPH_KEY_LEN + 16];
2175         u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2176         u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2177         u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2178         int session_key_len, con_secret_len;
2179         int payload_len;
2180         u64 global_id;
2181         int ret;
2182
2183         if (con->state != CEPH_CON_S_V2_AUTH) {
2184                 con->error_msg = "protocol error, unexpected auth_done";
2185                 return -EINVAL;
2186         }
2187
2188         ceph_decode_64_safe(&p, end, global_id, bad);
2189         ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2190         ceph_decode_32_safe(&p, end, payload_len, bad);
2191
2192         dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2193              __func__, con, global_id, con->v2.con_mode, payload_len);
2194
2195         mutex_unlock(&con->mutex);
2196         session_key_len = 0;
2197         con_secret_len = 0;
2198         ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2199                                          session_key, &session_key_len,
2200                                          con_secret, &con_secret_len);
2201         mutex_lock(&con->mutex);
2202         if (con->state != CEPH_CON_S_V2_AUTH) {
2203                 dout("%s con %p state changed to %d\n", __func__, con,
2204                      con->state);
2205                 ret = -EAGAIN;
2206                 goto out;
2207         }
2208
2209         dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2210         if (ret)
2211                 goto out;
2212
2213         ret = setup_crypto(con, session_key, session_key_len, con_secret,
2214                            con_secret_len);
2215         if (ret)
2216                 goto out;
2217
2218         reset_out_kvecs(con);
2219         ret = prepare_auth_signature(con);
2220         if (ret) {
2221                 pr_err("prepare_auth_signature failed: %d\n", ret);
2222                 goto out;
2223         }
2224
2225         con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2226
2227 out:
2228         memzero_explicit(session_key_buf, sizeof(session_key_buf));
2229         memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2230         return ret;
2231
2232 bad:
2233         pr_err("failed to decode auth_done\n");
2234         return -EINVAL;
2235 }
2236
2237 static int process_auth_signature(struct ceph_connection *con,
2238                                   void *p, void *end)
2239 {
2240         u8 hmac[SHA256_DIGEST_SIZE];
2241         int ret;
2242
2243         if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2244                 con->error_msg = "protocol error, unexpected auth_signature";
2245                 return -EINVAL;
2246         }
2247
2248         ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2249                           con->v2.out_sign_kvec_cnt, hmac);
2250         if (ret)
2251                 return ret;
2252
2253         ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2254         if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2255                 con->error_msg = "integrity error, bad auth signature";
2256                 return -EBADMSG;
2257         }
2258
2259         dout("%s con %p auth signature ok\n", __func__, con);
2260
2261         /* no reset_out_kvecs() as our auth_signature may still be pending */
2262         if (!con->v2.server_cookie) {
2263                 ret = prepare_client_ident(con);
2264                 if (ret) {
2265                         pr_err("prepare_client_ident failed: %d\n", ret);
2266                         return ret;
2267                 }
2268
2269                 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2270         } else {
2271                 ret = prepare_session_reconnect(con);
2272                 if (ret) {
2273                         pr_err("prepare_session_reconnect failed: %d\n", ret);
2274                         return ret;
2275                 }
2276
2277                 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2278         }
2279
2280         return 0;
2281
2282 bad:
2283         pr_err("failed to decode auth_signature\n");
2284         return -EINVAL;
2285 }
2286
2287 static int process_server_ident(struct ceph_connection *con,
2288                                 void *p, void *end)
2289 {
2290         struct ceph_client *client = from_msgr(con->msgr);
2291         u64 features, required_features;
2292         struct ceph_entity_addr addr;
2293         u64 global_seq;
2294         u64 global_id;
2295         u64 cookie;
2296         u64 flags;
2297         int ret;
2298
2299         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2300                 con->error_msg = "protocol error, unexpected server_ident";
2301                 return -EINVAL;
2302         }
2303
2304         ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2305         if (ret) {
2306                 pr_err("failed to decode server addrs: %d\n", ret);
2307                 return ret;
2308         }
2309
2310         ceph_decode_64_safe(&p, end, global_id, bad);
2311         ceph_decode_64_safe(&p, end, global_seq, bad);
2312         ceph_decode_64_safe(&p, end, features, bad);
2313         ceph_decode_64_safe(&p, end, required_features, bad);
2314         ceph_decode_64_safe(&p, end, flags, bad);
2315         ceph_decode_64_safe(&p, end, cookie, bad);
2316
2317         dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2318              __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2319              global_id, global_seq, features, required_features, flags, cookie);
2320
2321         /* is this who we intended to talk to? */
2322         if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2323                 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2324                        ceph_pr_addr(&con->peer_addr),
2325                        le32_to_cpu(con->peer_addr.nonce),
2326                        ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2327                 con->error_msg = "wrong peer at address";
2328                 return -EINVAL;
2329         }
2330
2331         if (client->required_features & ~features) {
2332                 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2333                        features, client->required_features & ~features);
2334                 con->error_msg = "missing required protocol features";
2335                 return -EINVAL;
2336         }
2337
2338         /*
2339          * Both name->type and name->num are set in ceph_con_open() but
2340          * name->num may be bogus in the initial monmap.  name->type is
2341          * verified in handle_hello().
2342          */
2343         WARN_ON(!con->peer_name.type);
2344         con->peer_name.num = cpu_to_le64(global_id);
2345         con->v2.peer_global_seq = global_seq;
2346         con->peer_features = features;
2347         WARN_ON(required_features & ~client->supported_features);
2348         con->v2.server_cookie = cookie;
2349
2350         if (flags & CEPH_MSG_CONNECT_LOSSY) {
2351                 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2352                 WARN_ON(con->v2.server_cookie);
2353         } else {
2354                 WARN_ON(!con->v2.server_cookie);
2355         }
2356
2357         clear_in_sign_kvecs(con);
2358         clear_out_sign_kvecs(con);
2359         free_conn_bufs(con);
2360         con->delay = 0;  /* reset backoff memory */
2361
2362         con->state = CEPH_CON_S_OPEN;
2363         con->v2.out_state = OUT_S_GET_NEXT;
2364         return 0;
2365
2366 bad:
2367         pr_err("failed to decode server_ident\n");
2368         return -EINVAL;
2369 }
2370
2371 static int process_ident_missing_features(struct ceph_connection *con,
2372                                           void *p, void *end)
2373 {
2374         struct ceph_client *client = from_msgr(con->msgr);
2375         u64 missing_features;
2376
2377         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2378                 con->error_msg = "protocol error, unexpected ident_missing_features";
2379                 return -EINVAL;
2380         }
2381
2382         ceph_decode_64_safe(&p, end, missing_features, bad);
2383         pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2384                client->supported_features, missing_features);
2385         con->error_msg = "missing required protocol features";
2386         return -EINVAL;
2387
2388 bad:
2389         pr_err("failed to decode ident_missing_features\n");
2390         return -EINVAL;
2391 }
2392
2393 static int process_session_reconnect_ok(struct ceph_connection *con,
2394                                         void *p, void *end)
2395 {
2396         u64 seq;
2397
2398         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2399                 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2400                 return -EINVAL;
2401         }
2402
2403         ceph_decode_64_safe(&p, end, seq, bad);
2404
2405         dout("%s con %p seq %llu\n", __func__, con, seq);
2406         ceph_con_discard_requeued(con, seq);
2407
2408         clear_in_sign_kvecs(con);
2409         clear_out_sign_kvecs(con);
2410         free_conn_bufs(con);
2411         con->delay = 0;  /* reset backoff memory */
2412
2413         con->state = CEPH_CON_S_OPEN;
2414         con->v2.out_state = OUT_S_GET_NEXT;
2415         return 0;
2416
2417 bad:
2418         pr_err("failed to decode session_reconnect_ok\n");
2419         return -EINVAL;
2420 }
2421
2422 static int process_session_retry(struct ceph_connection *con,
2423                                  void *p, void *end)
2424 {
2425         u64 connect_seq;
2426         int ret;
2427
2428         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2429                 con->error_msg = "protocol error, unexpected session_retry";
2430                 return -EINVAL;
2431         }
2432
2433         ceph_decode_64_safe(&p, end, connect_seq, bad);
2434
2435         dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2436         WARN_ON(connect_seq <= con->v2.connect_seq);
2437         con->v2.connect_seq = connect_seq + 1;
2438
2439         free_conn_bufs(con);
2440
2441         reset_out_kvecs(con);
2442         ret = prepare_session_reconnect(con);
2443         if (ret) {
2444                 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2445                 return ret;
2446         }
2447
2448         return 0;
2449
2450 bad:
2451         pr_err("failed to decode session_retry\n");
2452         return -EINVAL;
2453 }
2454
2455 static int process_session_retry_global(struct ceph_connection *con,
2456                                         void *p, void *end)
2457 {
2458         u64 global_seq;
2459         int ret;
2460
2461         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2462                 con->error_msg = "protocol error, unexpected session_retry_global";
2463                 return -EINVAL;
2464         }
2465
2466         ceph_decode_64_safe(&p, end, global_seq, bad);
2467
2468         dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2469         WARN_ON(global_seq <= con->v2.global_seq);
2470         con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2471
2472         free_conn_bufs(con);
2473
2474         reset_out_kvecs(con);
2475         ret = prepare_session_reconnect(con);
2476         if (ret) {
2477                 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2478                 return ret;
2479         }
2480
2481         return 0;
2482
2483 bad:
2484         pr_err("failed to decode session_retry_global\n");
2485         return -EINVAL;
2486 }
2487
2488 static int process_session_reset(struct ceph_connection *con,
2489                                  void *p, void *end)
2490 {
2491         bool full;
2492         int ret;
2493
2494         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2495                 con->error_msg = "protocol error, unexpected session_reset";
2496                 return -EINVAL;
2497         }
2498
2499         ceph_decode_8_safe(&p, end, full, bad);
2500         if (!full) {
2501                 con->error_msg = "protocol error, bad session_reset";
2502                 return -EINVAL;
2503         }
2504
2505         pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2506                 ceph_pr_addr(&con->peer_addr));
2507         ceph_con_reset_session(con);
2508
2509         mutex_unlock(&con->mutex);
2510         if (con->ops->peer_reset)
2511                 con->ops->peer_reset(con);
2512         mutex_lock(&con->mutex);
2513         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2514                 dout("%s con %p state changed to %d\n", __func__, con,
2515                      con->state);
2516                 return -EAGAIN;
2517         }
2518
2519         free_conn_bufs(con);
2520
2521         reset_out_kvecs(con);
2522         ret = prepare_client_ident(con);
2523         if (ret) {
2524                 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2525                 return ret;
2526         }
2527
2528         con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2529         return 0;
2530
2531 bad:
2532         pr_err("failed to decode session_reset\n");
2533         return -EINVAL;
2534 }
2535
2536 static int process_keepalive2_ack(struct ceph_connection *con,
2537                                   void *p, void *end)
2538 {
2539         if (con->state != CEPH_CON_S_OPEN) {
2540                 con->error_msg = "protocol error, unexpected keepalive2_ack";
2541                 return -EINVAL;
2542         }
2543
2544         ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2545         ceph_decode_timespec64(&con->last_keepalive_ack, p);
2546
2547         dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2548              con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2549
2550         return 0;
2551
2552 bad:
2553         pr_err("failed to decode keepalive2_ack\n");
2554         return -EINVAL;
2555 }
2556
2557 static int process_ack(struct ceph_connection *con, void *p, void *end)
2558 {
2559         u64 seq;
2560
2561         if (con->state != CEPH_CON_S_OPEN) {
2562                 con->error_msg = "protocol error, unexpected ack";
2563                 return -EINVAL;
2564         }
2565
2566         ceph_decode_64_safe(&p, end, seq, bad);
2567
2568         dout("%s con %p seq %llu\n", __func__, con, seq);
2569         ceph_con_discard_sent(con, seq);
2570         return 0;
2571
2572 bad:
2573         pr_err("failed to decode ack\n");
2574         return -EINVAL;
2575 }
2576
2577 static int process_control(struct ceph_connection *con, void *p, void *end)
2578 {
2579         int tag = con->v2.in_desc.fd_tag;
2580         int ret;
2581
2582         dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2583
2584         switch (tag) {
2585         case FRAME_TAG_HELLO:
2586                 ret = process_hello(con, p, end);
2587                 break;
2588         case FRAME_TAG_AUTH_BAD_METHOD:
2589                 ret = process_auth_bad_method(con, p, end);
2590                 break;
2591         case FRAME_TAG_AUTH_REPLY_MORE:
2592                 ret = process_auth_reply_more(con, p, end);
2593                 break;
2594         case FRAME_TAG_AUTH_DONE:
2595                 ret = process_auth_done(con, p, end);
2596                 break;
2597         case FRAME_TAG_AUTH_SIGNATURE:
2598                 ret = process_auth_signature(con, p, end);
2599                 break;
2600         case FRAME_TAG_SERVER_IDENT:
2601                 ret = process_server_ident(con, p, end);
2602                 break;
2603         case FRAME_TAG_IDENT_MISSING_FEATURES:
2604                 ret = process_ident_missing_features(con, p, end);
2605                 break;
2606         case FRAME_TAG_SESSION_RECONNECT_OK:
2607                 ret = process_session_reconnect_ok(con, p, end);
2608                 break;
2609         case FRAME_TAG_SESSION_RETRY:
2610                 ret = process_session_retry(con, p, end);
2611                 break;
2612         case FRAME_TAG_SESSION_RETRY_GLOBAL:
2613                 ret = process_session_retry_global(con, p, end);
2614                 break;
2615         case FRAME_TAG_SESSION_RESET:
2616                 ret = process_session_reset(con, p, end);
2617                 break;
2618         case FRAME_TAG_KEEPALIVE2_ACK:
2619                 ret = process_keepalive2_ack(con, p, end);
2620                 break;
2621         case FRAME_TAG_ACK:
2622                 ret = process_ack(con, p, end);
2623                 break;
2624         default:
2625                 pr_err("bad tag %d\n", tag);
2626                 con->error_msg = "protocol error, bad tag";
2627                 return -EINVAL;
2628         }
2629         if (ret) {
2630                 dout("%s con %p error %d\n", __func__, con, ret);
2631                 return ret;
2632         }
2633
2634         prepare_read_preamble(con);
2635         return 0;
2636 }
2637
2638 /*
2639  * Return:
2640  *   1 - con->in_msg set, read message
2641  *   0 - skip message
2642  *  <0 - error
2643  */
2644 static int process_message_header(struct ceph_connection *con,
2645                                   void *p, void *end)
2646 {
2647         struct ceph_frame_desc *desc = &con->v2.in_desc;
2648         struct ceph_msg_header2 *hdr2 = p;
2649         struct ceph_msg_header hdr;
2650         int skip;
2651         int ret;
2652         u64 seq;
2653
2654         /* verify seq# */
2655         seq = le64_to_cpu(hdr2->seq);
2656         if ((s64)seq - (s64)con->in_seq < 1) {
2657                 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2658                         ENTITY_NAME(con->peer_name),
2659                         ceph_pr_addr(&con->peer_addr),
2660                         seq, con->in_seq + 1);
2661                 return 0;
2662         }
2663         if ((s64)seq - (s64)con->in_seq > 1) {
2664                 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2665                 con->error_msg = "bad message sequence # for incoming message";
2666                 return -EBADE;
2667         }
2668
2669         ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2670
2671         fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2672                     desc->fd_lens[3], &con->peer_name);
2673         ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2674         if (ret)
2675                 return ret;
2676
2677         WARN_ON(!con->in_msg ^ skip);
2678         if (skip)
2679                 return 0;
2680
2681         WARN_ON(!con->in_msg);
2682         WARN_ON(con->in_msg->con != con);
2683         return 1;
2684 }
2685
2686 static int process_message(struct ceph_connection *con)
2687 {
2688         ceph_con_process_message(con);
2689
2690         /*
2691          * We could have been closed by ceph_con_close() because
2692          * ceph_con_process_message() temporarily drops con->mutex.
2693          */
2694         if (con->state != CEPH_CON_S_OPEN) {
2695                 dout("%s con %p state changed to %d\n", __func__, con,
2696                      con->state);
2697                 return -EAGAIN;
2698         }
2699
2700         prepare_read_preamble(con);
2701         return 0;
2702 }
2703
2704 static int __handle_control(struct ceph_connection *con, void *p)
2705 {
2706         void *end = p + con->v2.in_desc.fd_lens[0];
2707         struct ceph_msg *msg;
2708         int ret;
2709
2710         if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2711                 return process_control(con, p, end);
2712
2713         ret = process_message_header(con, p, end);
2714         if (ret < 0)
2715                 return ret;
2716         if (ret == 0) {
2717                 prepare_skip_message(con);
2718                 return 0;
2719         }
2720
2721         msg = con->in_msg;  /* set in process_message_header() */
2722         if (front_len(msg)) {
2723                 WARN_ON(front_len(msg) > msg->front_alloc_len);
2724                 msg->front.iov_len = front_len(msg);
2725         } else {
2726                 msg->front.iov_len = 0;
2727         }
2728         if (middle_len(msg)) {
2729                 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2730                 msg->middle->vec.iov_len = middle_len(msg);
2731         } else if (msg->middle) {
2732                 msg->middle->vec.iov_len = 0;
2733         }
2734
2735         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2736                 return process_message(con);
2737
2738         if (con_secure(con))
2739                 return prepare_read_tail_secure(con);
2740
2741         return prepare_read_tail_plain(con);
2742 }
2743
2744 static int handle_preamble(struct ceph_connection *con)
2745 {
2746         struct ceph_frame_desc *desc = &con->v2.in_desc;
2747         int ret;
2748
2749         if (con_secure(con)) {
2750                 ret = decrypt_preamble(con);
2751                 if (ret) {
2752                         if (ret == -EBADMSG)
2753                                 con->error_msg = "integrity error, bad preamble auth tag";
2754                         return ret;
2755                 }
2756         }
2757
2758         ret = decode_preamble(con->v2.in_buf, desc);
2759         if (ret) {
2760                 if (ret == -EBADMSG)
2761                         con->error_msg = "integrity error, bad crc";
2762                 else
2763                         con->error_msg = "protocol error, bad preamble";
2764                 return ret;
2765         }
2766
2767         dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2768              con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2769              desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2770
2771         if (!con_secure(con))
2772                 return prepare_read_control(con);
2773
2774         if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2775                 return prepare_read_control_remainder(con);
2776
2777         return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2778 }
2779
2780 static int handle_control(struct ceph_connection *con)
2781 {
2782         int ctrl_len = con->v2.in_desc.fd_lens[0];
2783         void *buf;
2784         int ret;
2785
2786         WARN_ON(con_secure(con));
2787
2788         ret = verify_control_crc(con);
2789         if (ret) {
2790                 con->error_msg = "integrity error, bad crc";
2791                 return ret;
2792         }
2793
2794         if (con->state == CEPH_CON_S_V2_AUTH) {
2795                 buf = alloc_conn_buf(con, ctrl_len);
2796                 if (!buf)
2797                         return -ENOMEM;
2798
2799                 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2800                 return __handle_control(con, buf);
2801         }
2802
2803         return __handle_control(con, con->v2.in_kvecs[0].iov_base);
2804 }
2805
2806 static int handle_control_remainder(struct ceph_connection *con)
2807 {
2808         int ret;
2809
2810         WARN_ON(!con_secure(con));
2811
2812         ret = decrypt_control_remainder(con);
2813         if (ret) {
2814                 if (ret == -EBADMSG)
2815                         con->error_msg = "integrity error, bad control remainder auth tag";
2816                 return ret;
2817         }
2818
2819         return __handle_control(con, con->v2.in_kvecs[0].iov_base -
2820                                      CEPH_PREAMBLE_INLINE_LEN);
2821 }
2822
2823 static int handle_epilogue(struct ceph_connection *con)
2824 {
2825         u32 front_crc, middle_crc, data_crc;
2826         int ret;
2827
2828         if (con_secure(con)) {
2829                 ret = decrypt_tail(con);
2830                 if (ret) {
2831                         if (ret == -EBADMSG)
2832                                 con->error_msg = "integrity error, bad epilogue auth tag";
2833                         return ret;
2834                 }
2835
2836                 /* just late_status */
2837                 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
2838                 if (ret) {
2839                         con->error_msg = "protocol error, bad epilogue";
2840                         return ret;
2841                 }
2842         } else {
2843                 ret = decode_epilogue(con->v2.in_buf, &front_crc,
2844                                       &middle_crc, &data_crc);
2845                 if (ret) {
2846                         con->error_msg = "protocol error, bad epilogue";
2847                         return ret;
2848                 }
2849
2850                 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
2851                                            data_crc);
2852                 if (ret) {
2853                         con->error_msg = "integrity error, bad crc";
2854                         return ret;
2855                 }
2856         }
2857
2858         return process_message(con);
2859 }
2860
2861 static void finish_skip(struct ceph_connection *con)
2862 {
2863         dout("%s con %p\n", __func__, con);
2864
2865         if (con_secure(con))
2866                 gcm_inc_nonce(&con->v2.in_gcm_nonce);
2867
2868         __finish_skip(con);
2869 }
2870
2871 static int populate_in_iter(struct ceph_connection *con)
2872 {
2873         int ret;
2874
2875         dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
2876              con->v2.in_state);
2877         WARN_ON(iov_iter_count(&con->v2.in_iter));
2878
2879         if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
2880                 ret = process_banner_prefix(con);
2881         } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
2882                 ret = process_banner_payload(con);
2883         } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
2884                     con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
2885                    con->state == CEPH_CON_S_OPEN) {
2886                 switch (con->v2.in_state) {
2887                 case IN_S_HANDLE_PREAMBLE:
2888                         ret = handle_preamble(con);
2889                         break;
2890                 case IN_S_HANDLE_CONTROL:
2891                         ret = handle_control(con);
2892                         break;
2893                 case IN_S_HANDLE_CONTROL_REMAINDER:
2894                         ret = handle_control_remainder(con);
2895                         break;
2896                 case IN_S_PREPARE_READ_DATA:
2897                         ret = prepare_read_data(con);
2898                         break;
2899                 case IN_S_PREPARE_READ_DATA_CONT:
2900                         prepare_read_data_cont(con);
2901                         ret = 0;
2902                         break;
2903                 case IN_S_PREPARE_READ_ENC_PAGE:
2904                         prepare_read_enc_page(con);
2905                         ret = 0;
2906                         break;
2907                 case IN_S_HANDLE_EPILOGUE:
2908                         ret = handle_epilogue(con);
2909                         break;
2910                 case IN_S_FINISH_SKIP:
2911                         finish_skip(con);
2912                         ret = 0;
2913                         break;
2914                 default:
2915                         WARN(1, "bad in_state %d", con->v2.in_state);
2916                         return -EINVAL;
2917                 }
2918         } else {
2919                 WARN(1, "bad state %d", con->state);
2920                 return -EINVAL;
2921         }
2922         if (ret) {
2923                 dout("%s con %p error %d\n", __func__, con, ret);
2924                 return ret;
2925         }
2926
2927         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2928                 return -ENODATA;
2929         dout("%s con %p populated %zu\n", __func__, con,
2930              iov_iter_count(&con->v2.in_iter));
2931         return 1;
2932 }
2933
2934 int ceph_con_v2_try_read(struct ceph_connection *con)
2935 {
2936         int ret;
2937
2938         dout("%s con %p state %d need %zu\n", __func__, con, con->state,
2939              iov_iter_count(&con->v2.in_iter));
2940
2941         if (con->state == CEPH_CON_S_PREOPEN)
2942                 return 0;
2943
2944         /*
2945          * We should always have something pending here.  If not,
2946          * avoid calling populate_in_iter() as if we read something
2947          * (ceph_tcp_recv() would immediately return 1).
2948          */
2949         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2950                 return -ENODATA;
2951
2952         for (;;) {
2953                 ret = ceph_tcp_recv(con);
2954                 if (ret <= 0)
2955                         return ret;
2956
2957                 ret = populate_in_iter(con);
2958                 if (ret <= 0) {
2959                         if (ret && ret != -EAGAIN && !con->error_msg)
2960                                 con->error_msg = "read processing error";
2961                         return ret;
2962                 }
2963         }
2964 }
2965
2966 static void queue_data(struct ceph_connection *con)
2967 {
2968         struct bio_vec bv;
2969
2970         con->v2.out_epil.data_crc = -1;
2971         ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
2972                                   data_len(con->out_msg));
2973
2974         get_bvec_at(&con->v2.out_cursor, &bv);
2975         set_out_bvec(con, &bv, true);
2976         con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
2977 }
2978
2979 static void queue_data_cont(struct ceph_connection *con)
2980 {
2981         struct bio_vec bv;
2982
2983         con->v2.out_epil.data_crc = ceph_crc32c_page(
2984                 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
2985                 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
2986
2987         ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
2988         if (con->v2.out_cursor.total_resid) {
2989                 get_bvec_at(&con->v2.out_cursor, &bv);
2990                 set_out_bvec(con, &bv, true);
2991                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
2992                 return;
2993         }
2994
2995         /*
2996          * We've written all data.  Queue epilogue.  Once it's written,
2997          * we are done.
2998          */
2999         reset_out_kvecs(con);
3000         prepare_epilogue_plain(con, false);
3001         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3002 }
3003
3004 static void queue_enc_page(struct ceph_connection *con)
3005 {
3006         struct bio_vec bv;
3007
3008         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3009              con->v2.out_enc_resid);
3010         WARN_ON(!con->v2.out_enc_resid);
3011
3012         bv.bv_page = con->v2.out_enc_pages[con->v2.out_enc_i];
3013         bv.bv_offset = 0;
3014         bv.bv_len = min(con->v2.out_enc_resid, (int)PAGE_SIZE);
3015
3016         set_out_bvec(con, &bv, false);
3017         con->v2.out_enc_i++;
3018         con->v2.out_enc_resid -= bv.bv_len;
3019
3020         if (con->v2.out_enc_resid) {
3021                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3022                 return;
3023         }
3024
3025         /*
3026          * We've queued the last piece of ciphertext (ending with
3027          * epilogue) + auth tag.  Once it's written, we are done.
3028          */
3029         WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3030         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3031 }
3032
3033 static void queue_zeros(struct ceph_connection *con)
3034 {
3035         dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3036
3037         if (con->v2.out_zero) {
3038                 set_out_bvec_zero(con);
3039                 con->v2.out_zero -= con->v2.out_bvec.bv_len;
3040                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3041                 return;
3042         }
3043
3044         /*
3045          * We've zero-filled everything up to epilogue.  Queue epilogue
3046          * with late_status set to ABORTED and crcs adjusted for zeros.
3047          * Once it's written, we are done patching up for the revoke.
3048          */
3049         reset_out_kvecs(con);
3050         prepare_epilogue_plain(con, true);
3051         con->v2.out_state = OUT_S_FINISH_MESSAGE;
3052 }
3053
3054 static void finish_message(struct ceph_connection *con)
3055 {
3056         dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3057
3058         /* we end up here both plain and secure modes */
3059         if (con->v2.out_enc_pages) {
3060                 WARN_ON(!con->v2.out_enc_page_cnt);
3061                 ceph_release_page_vector(con->v2.out_enc_pages,
3062                                          con->v2.out_enc_page_cnt);
3063                 con->v2.out_enc_pages = NULL;
3064                 con->v2.out_enc_page_cnt = 0;
3065         }
3066         /* message may have been revoked */
3067         if (con->out_msg) {
3068                 ceph_msg_put(con->out_msg);
3069                 con->out_msg = NULL;
3070         }
3071
3072         con->v2.out_state = OUT_S_GET_NEXT;
3073 }
3074
3075 static int populate_out_iter(struct ceph_connection *con)
3076 {
3077         int ret;
3078
3079         dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3080              con->v2.out_state);
3081         WARN_ON(iov_iter_count(&con->v2.out_iter));
3082
3083         if (con->state != CEPH_CON_S_OPEN) {
3084                 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3085                         con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3086                 goto nothing_pending;
3087         }
3088
3089         switch (con->v2.out_state) {
3090         case OUT_S_QUEUE_DATA:
3091                 WARN_ON(!con->out_msg);
3092                 queue_data(con);
3093                 goto populated;
3094         case OUT_S_QUEUE_DATA_CONT:
3095                 WARN_ON(!con->out_msg);
3096                 queue_data_cont(con);
3097                 goto populated;
3098         case OUT_S_QUEUE_ENC_PAGE:
3099                 queue_enc_page(con);
3100                 goto populated;
3101         case OUT_S_QUEUE_ZEROS:
3102                 WARN_ON(con->out_msg);  /* revoked */
3103                 queue_zeros(con);
3104                 goto populated;
3105         case OUT_S_FINISH_MESSAGE:
3106                 finish_message(con);
3107                 break;
3108         case OUT_S_GET_NEXT:
3109                 break;
3110         default:
3111                 WARN(1, "bad out_state %d", con->v2.out_state);
3112                 return -EINVAL;
3113         }
3114
3115         WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3116         if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3117                 ret = prepare_keepalive2(con);
3118                 if (ret) {
3119                         pr_err("prepare_keepalive2 failed: %d\n", ret);
3120                         return ret;
3121                 }
3122         } else if (!list_empty(&con->out_queue)) {
3123                 ceph_con_get_out_msg(con);
3124                 ret = prepare_message(con);
3125                 if (ret) {
3126                         pr_err("prepare_message failed: %d\n", ret);
3127                         return ret;
3128                 }
3129         } else if (con->in_seq > con->in_seq_acked) {
3130                 ret = prepare_ack(con);
3131                 if (ret) {
3132                         pr_err("prepare_ack failed: %d\n", ret);
3133                         return ret;
3134                 }
3135         } else {
3136                 goto nothing_pending;
3137         }
3138
3139 populated:
3140         if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3141                 return -ENODATA;
3142         dout("%s con %p populated %zu\n", __func__, con,
3143              iov_iter_count(&con->v2.out_iter));
3144         return 1;
3145
3146 nothing_pending:
3147         WARN_ON(iov_iter_count(&con->v2.out_iter));
3148         dout("%s con %p nothing pending\n", __func__, con);
3149         ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3150         return 0;
3151 }
3152
3153 int ceph_con_v2_try_write(struct ceph_connection *con)
3154 {
3155         int ret;
3156
3157         dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3158              iov_iter_count(&con->v2.out_iter));
3159
3160         /* open the socket first? */
3161         if (con->state == CEPH_CON_S_PREOPEN) {
3162                 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3163
3164                 /*
3165                  * Always bump global_seq.  Bump connect_seq only if
3166                  * there is a session (i.e. we are reconnecting and will
3167                  * send session_reconnect instead of client_ident).
3168                  */
3169                 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3170                 if (con->v2.server_cookie)
3171                         con->v2.connect_seq++;
3172
3173                 ret = prepare_read_banner_prefix(con);
3174                 if (ret) {
3175                         pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3176                         con->error_msg = "connect error";
3177                         return ret;
3178                 }
3179
3180                 reset_out_kvecs(con);
3181                 ret = prepare_banner(con);
3182                 if (ret) {
3183                         pr_err("prepare_banner failed: %d\n", ret);
3184                         con->error_msg = "connect error";
3185                         return ret;
3186                 }
3187
3188                 ret = ceph_tcp_connect(con);
3189                 if (ret) {
3190                         pr_err("ceph_tcp_connect failed: %d\n", ret);
3191                         con->error_msg = "connect error";
3192                         return ret;
3193                 }
3194         }
3195
3196         if (!iov_iter_count(&con->v2.out_iter)) {
3197                 ret = populate_out_iter(con);
3198                 if (ret <= 0) {
3199                         if (ret && ret != -EAGAIN && !con->error_msg)
3200                                 con->error_msg = "write processing error";
3201                         return ret;
3202                 }
3203         }
3204
3205         tcp_sock_set_cork(con->sock->sk, true);
3206         for (;;) {
3207                 ret = ceph_tcp_send(con);
3208                 if (ret <= 0)
3209                         break;
3210
3211                 ret = populate_out_iter(con);
3212                 if (ret <= 0) {
3213                         if (ret && ret != -EAGAIN && !con->error_msg)
3214                                 con->error_msg = "write processing error";
3215                         break;
3216                 }
3217         }
3218
3219         tcp_sock_set_cork(con->sock->sk, false);
3220         return ret;
3221 }
3222
3223 static u32 crc32c_zeros(u32 crc, int zero_len)
3224 {
3225         int len;
3226
3227         while (zero_len) {
3228                 len = min(zero_len, (int)PAGE_SIZE);
3229                 crc = crc32c(crc, page_address(ceph_zero_page), len);
3230                 zero_len -= len;
3231         }
3232
3233         return crc;
3234 }
3235
3236 static void prepare_zero_front(struct ceph_connection *con, int resid)
3237 {
3238         int sent;
3239
3240         WARN_ON(!resid || resid > front_len(con->out_msg));
3241         sent = front_len(con->out_msg) - resid;
3242         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3243
3244         if (sent) {
3245                 con->v2.out_epil.front_crc =
3246                         crc32c(-1, con->out_msg->front.iov_base, sent);
3247                 con->v2.out_epil.front_crc =
3248                         crc32c_zeros(con->v2.out_epil.front_crc, resid);
3249         } else {
3250                 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3251         }
3252
3253         con->v2.out_iter.count -= resid;
3254         out_zero_add(con, resid);
3255 }
3256
3257 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3258 {
3259         int sent;
3260
3261         WARN_ON(!resid || resid > middle_len(con->out_msg));
3262         sent = middle_len(con->out_msg) - resid;
3263         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3264
3265         if (sent) {
3266                 con->v2.out_epil.middle_crc =
3267                         crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3268                 con->v2.out_epil.middle_crc =
3269                         crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3270         } else {
3271                 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3272         }
3273
3274         con->v2.out_iter.count -= resid;
3275         out_zero_add(con, resid);
3276 }
3277
3278 static void prepare_zero_data(struct ceph_connection *con)
3279 {
3280         dout("%s con %p\n", __func__, con);
3281         con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3282         out_zero_add(con, data_len(con->out_msg));
3283 }
3284
3285 static void revoke_at_queue_data(struct ceph_connection *con)
3286 {
3287         int boundary;
3288         int resid;
3289
3290         WARN_ON(!data_len(con->out_msg));
3291         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3292         resid = iov_iter_count(&con->v2.out_iter);
3293
3294         boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3295         if (resid > boundary) {
3296                 resid -= boundary;
3297                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3298                 dout("%s con %p was sending head\n", __func__, con);
3299                 if (front_len(con->out_msg))
3300                         prepare_zero_front(con, front_len(con->out_msg));
3301                 if (middle_len(con->out_msg))
3302                         prepare_zero_middle(con, middle_len(con->out_msg));
3303                 prepare_zero_data(con);
3304                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3305                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3306                 return;
3307         }
3308
3309         boundary = middle_len(con->out_msg);
3310         if (resid > boundary) {
3311                 resid -= boundary;
3312                 dout("%s con %p was sending front\n", __func__, con);
3313                 prepare_zero_front(con, resid);
3314                 if (middle_len(con->out_msg))
3315                         prepare_zero_middle(con, middle_len(con->out_msg));
3316                 prepare_zero_data(con);
3317                 queue_zeros(con);
3318                 return;
3319         }
3320
3321         WARN_ON(!resid);
3322         dout("%s con %p was sending middle\n", __func__, con);
3323         prepare_zero_middle(con, resid);
3324         prepare_zero_data(con);
3325         queue_zeros(con);
3326 }
3327
3328 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3329 {
3330         int sent, resid;  /* current piece of data */
3331
3332         WARN_ON(!data_len(con->out_msg));
3333         WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3334         resid = iov_iter_count(&con->v2.out_iter);
3335         WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3336         sent = con->v2.out_bvec.bv_len - resid;
3337         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3338
3339         if (sent) {
3340                 con->v2.out_epil.data_crc = ceph_crc32c_page(
3341                         con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3342                         con->v2.out_bvec.bv_offset, sent);
3343                 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3344         }
3345         WARN_ON(resid > con->v2.out_cursor.total_resid);
3346         con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3347                                                 con->v2.out_cursor.total_resid);
3348
3349         con->v2.out_iter.count -= resid;
3350         out_zero_add(con, con->v2.out_cursor.total_resid);
3351         queue_zeros(con);
3352 }
3353
3354 static void revoke_at_finish_message(struct ceph_connection *con)
3355 {
3356         int boundary;
3357         int resid;
3358
3359         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3360         resid = iov_iter_count(&con->v2.out_iter);
3361
3362         if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3363             !data_len(con->out_msg)) {
3364                 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3365                 dout("%s con %p was sending head (empty message) - noop\n",
3366                      __func__, con);
3367                 return;
3368         }
3369
3370         boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3371                    CEPH_EPILOGUE_PLAIN_LEN;
3372         if (resid > boundary) {
3373                 resid -= boundary;
3374                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3375                 dout("%s con %p was sending head\n", __func__, con);
3376                 if (front_len(con->out_msg))
3377                         prepare_zero_front(con, front_len(con->out_msg));
3378                 if (middle_len(con->out_msg))
3379                         prepare_zero_middle(con, middle_len(con->out_msg));
3380                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3381                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3382                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3383                 return;
3384         }
3385
3386         boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3387         if (resid > boundary) {
3388                 resid -= boundary;
3389                 dout("%s con %p was sending front\n", __func__, con);
3390                 prepare_zero_front(con, resid);
3391                 if (middle_len(con->out_msg))
3392                         prepare_zero_middle(con, middle_len(con->out_msg));
3393                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3394                 queue_zeros(con);
3395                 return;
3396         }
3397
3398         boundary = CEPH_EPILOGUE_PLAIN_LEN;
3399         if (resid > boundary) {
3400                 resid -= boundary;
3401                 dout("%s con %p was sending middle\n", __func__, con);
3402                 prepare_zero_middle(con, resid);
3403                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3404                 queue_zeros(con);
3405                 return;
3406         }
3407
3408         WARN_ON(!resid);
3409         dout("%s con %p was sending epilogue - noop\n", __func__, con);
3410 }
3411
3412 void ceph_con_v2_revoke(struct ceph_connection *con)
3413 {
3414         WARN_ON(con->v2.out_zero);
3415
3416         if (con_secure(con)) {
3417                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3418                         con->v2.out_state != OUT_S_FINISH_MESSAGE);
3419                 dout("%s con %p secure - noop\n", __func__, con);
3420                 return;
3421         }
3422
3423         switch (con->v2.out_state) {
3424         case OUT_S_QUEUE_DATA:
3425                 revoke_at_queue_data(con);
3426                 break;
3427         case OUT_S_QUEUE_DATA_CONT:
3428                 revoke_at_queue_data_cont(con);
3429                 break;
3430         case OUT_S_FINISH_MESSAGE:
3431                 revoke_at_finish_message(con);
3432                 break;
3433         default:
3434                 WARN(1, "bad out_state %d", con->v2.out_state);
3435                 break;
3436         }
3437 }
3438
3439 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3440 {
3441         int remaining;
3442         int resid;
3443
3444         WARN_ON(con_secure(con));
3445         WARN_ON(!data_len(con->in_msg));
3446         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3447         resid = iov_iter_count(&con->v2.in_iter);
3448         WARN_ON(!resid);
3449
3450         remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3451         dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3452              remaining);
3453         con->v2.in_iter.count -= resid;
3454         set_in_skip(con, resid + remaining);
3455         con->v2.in_state = IN_S_FINISH_SKIP;
3456 }
3457
3458 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3459 {
3460         int recved, resid;  /* current piece of data */
3461         int remaining;
3462
3463         WARN_ON(con_secure(con));
3464         WARN_ON(!data_len(con->in_msg));
3465         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3466         resid = iov_iter_count(&con->v2.in_iter);
3467         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3468         recved = con->v2.in_bvec.bv_len - resid;
3469         dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3470
3471         if (recved)
3472                 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3473         WARN_ON(resid > con->v2.in_cursor.total_resid);
3474
3475         remaining = CEPH_EPILOGUE_PLAIN_LEN;
3476         dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3477              con->v2.in_cursor.total_resid, remaining);
3478         con->v2.in_iter.count -= resid;
3479         set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3480         con->v2.in_state = IN_S_FINISH_SKIP;
3481 }
3482
3483 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3484 {
3485         int resid;  /* current enc page (not necessarily data) */
3486
3487         WARN_ON(!con_secure(con));
3488         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3489         resid = iov_iter_count(&con->v2.in_iter);
3490         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3491
3492         dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3493              con->v2.in_enc_resid);
3494         con->v2.in_iter.count -= resid;
3495         set_in_skip(con, resid + con->v2.in_enc_resid);
3496         con->v2.in_state = IN_S_FINISH_SKIP;
3497 }
3498
3499 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3500 {
3501         int resid;
3502
3503         resid = iov_iter_count(&con->v2.in_iter);
3504         WARN_ON(!resid);
3505
3506         dout("%s con %p resid %d\n", __func__, con, resid);
3507         con->v2.in_iter.count -= resid;
3508         set_in_skip(con, resid);
3509         con->v2.in_state = IN_S_FINISH_SKIP;
3510 }
3511
3512 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3513 {
3514         switch (con->v2.in_state) {
3515         case IN_S_PREPARE_READ_DATA:
3516                 revoke_at_prepare_read_data(con);
3517                 break;
3518         case IN_S_PREPARE_READ_DATA_CONT:
3519                 revoke_at_prepare_read_data_cont(con);
3520                 break;
3521         case IN_S_PREPARE_READ_ENC_PAGE:
3522                 revoke_at_prepare_read_enc_page(con);
3523                 break;
3524         case IN_S_HANDLE_EPILOGUE:
3525                 revoke_at_handle_epilogue(con);
3526                 break;
3527         default:
3528                 WARN(1, "bad in_state %d", con->v2.in_state);
3529                 break;
3530         }
3531 }
3532
3533 bool ceph_con_v2_opened(struct ceph_connection *con)
3534 {
3535         return con->v2.peer_global_seq;
3536 }
3537
3538 void ceph_con_v2_reset_session(struct ceph_connection *con)
3539 {
3540         con->v2.client_cookie = 0;
3541         con->v2.server_cookie = 0;
3542         con->v2.global_seq = 0;
3543         con->v2.connect_seq = 0;
3544         con->v2.peer_global_seq = 0;
3545 }
3546
3547 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3548 {
3549         iov_iter_truncate(&con->v2.in_iter, 0);
3550         iov_iter_truncate(&con->v2.out_iter, 0);
3551         con->v2.out_zero = 0;
3552
3553         clear_in_sign_kvecs(con);
3554         clear_out_sign_kvecs(con);
3555         free_conn_bufs(con);
3556
3557         if (con->v2.in_enc_pages) {
3558                 WARN_ON(!con->v2.in_enc_page_cnt);
3559                 ceph_release_page_vector(con->v2.in_enc_pages,
3560                                          con->v2.in_enc_page_cnt);
3561                 con->v2.in_enc_pages = NULL;
3562                 con->v2.in_enc_page_cnt = 0;
3563         }
3564         if (con->v2.out_enc_pages) {
3565                 WARN_ON(!con->v2.out_enc_page_cnt);
3566                 ceph_release_page_vector(con->v2.out_enc_pages,
3567                                          con->v2.out_enc_page_cnt);
3568                 con->v2.out_enc_pages = NULL;
3569                 con->v2.out_enc_page_cnt = 0;
3570         }
3571
3572         con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3573         memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3574         memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3575
3576         if (con->v2.hmac_tfm) {
3577                 crypto_free_shash(con->v2.hmac_tfm);
3578                 con->v2.hmac_tfm = NULL;
3579         }
3580         if (con->v2.gcm_req) {
3581                 aead_request_free(con->v2.gcm_req);
3582                 con->v2.gcm_req = NULL;
3583         }
3584         if (con->v2.gcm_tfm) {
3585                 crypto_free_aead(con->v2.gcm_tfm);
3586                 con->v2.gcm_tfm = NULL;
3587         }
3588 }